{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_steps:  1600\n",
      "device:  cuda:0\n",
      "state dim:  24\n",
      "action dim:  Box(4,)\n",
      "leraning rate:  0.0001\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import torch\n",
    "from sac_agent import soft_actor_critic_agent\n",
    "from replay_memory import ReplayMemory\n",
    "\n",
    "seed=0  \n",
    "env = gym.make('BipedalWalker-v2')\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "env.seed(seed)\n",
    "max_steps = env._max_episode_steps\n",
    "print('max_steps: ', max_steps)\n",
    "\n",
    "batch_size=256\n",
    "\n",
    "LEARNING_RATE=0.0001\n",
    "eval=True  ## \n",
    "start_steps=10000 ## Steps sampling random actions\n",
    "replay_size=1000000 ## size of replay buffer\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# alpha=0.2  # relative importance of the entropy\n",
    "# gamma=0.99  # discount factor \n",
    "# tau=0.005  # target smoothing coefficient(τ)\n",
    "\n",
    "agent = soft_actor_critic_agent(env.observation_space.shape[0], env.action_space, \\\n",
    "        device=device, hidden_size=256, lr=LEARNING_RATE, gamma=0.99, tau=0.005, alpha=0.2)\n",
    "\n",
    "memory = ReplayMemory(replay_size)\n",
    "\n",
    "print('device: ', device)\n",
    "print('state dim: ', env.observation_space.shape[0])\n",
    "print('action dim: ', env.action_space)\n",
    "print('leraning rate: ', LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(agent, directory, filename, suffix):\n",
    "    torch.save(agent.policy.state_dict(), '%s/%s_actor_%s.pth' % (directory, filename, suffix))\n",
    "    torch.save(agent.critic.state_dict(), '%s/%s_critic_%s.pth' % (directory, filename, suffix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 0, Total Steps: 58, Ep.Steps: 58, Score: -116.42, Avg.Score: -116.42, Time: 00:00:00\n",
      "Ep.: 1, Total Steps: 133, Ep.Steps: 75, Score: -100.36, Avg.Score: -108.39, Time: 00:00:00\n",
      "Ep.: 2, Total Steps: 190, Ep.Steps: 57, Score: -113.98, Avg.Score: -110.25, Time: 00:00:00\n",
      "Ep.: 3, Total Steps: 1790, Ep.Steps: 1600, Score: -86.24, Avg.Score: -104.25, Time: 00:01:23\n",
      "Ep.: 4, Total Steps: 1884, Ep.Steps: 94, Score: -119.78, Avg.Score: -107.35, Time: 00:01:29\n",
      "Ep.: 5, Total Steps: 1981, Ep.Steps: 97, Score: -99.84, Avg.Score: -106.10, Time: 00:01:34\n",
      "Ep.: 6, Total Steps: 3581, Ep.Steps: 1600, Score: -78.23, Avg.Score: -102.12, Time: 00:03:01\n",
      "Ep.: 7, Total Steps: 3679, Ep.Steps: 98, Score: -107.01, Avg.Score: -102.73, Time: 00:03:06\n",
      "Ep.: 8, Total Steps: 3761, Ep.Steps: 82, Score: -101.43, Avg.Score: -102.59, Time: 00:03:11\n",
      "Ep.: 9, Total Steps: 3879, Ep.Steps: 118, Score: -101.66, Avg.Score: -102.49, Time: 00:03:17\n",
      "Ep.: 10, Total Steps: 3919, Ep.Steps: 40, Score: -112.11, Avg.Score: -103.37, Time: 00:03:20\n",
      "Ep.: 11, Total Steps: 4049, Ep.Steps: 130, Score: -100.36, Avg.Score: -103.12, Time: 00:03:27\n",
      "Ep.: 12, Total Steps: 4114, Ep.Steps: 65, Score: -115.66, Avg.Score: -104.08, Time: 00:03:30\n",
      "Ep.: 13, Total Steps: 4215, Ep.Steps: 101, Score: -101.97, Avg.Score: -103.93, Time: 00:03:36\n",
      "Ep.: 14, Total Steps: 5815, Ep.Steps: 1600, Score: -81.33, Avg.Score: -102.42, Time: 00:05:03\n",
      "Ep.: 15, Total Steps: 7415, Ep.Steps: 1600, Score: -88.84, Avg.Score: -101.58, Time: 00:06:29\n",
      "Ep.: 16, Total Steps: 9015, Ep.Steps: 1600, Score: -79.01, Avg.Score: -100.25, Time: 00:07:57\n",
      "Ep.: 17, Total Steps: 9108, Ep.Steps: 93, Score: -99.30, Avg.Score: -100.20, Time: 00:08:02\n",
      "Ep.: 18, Total Steps: 9266, Ep.Steps: 158, Score: -120.68, Avg.Score: -101.27, Time: 00:08:10\n",
      "Ep.: 19, Total Steps: 9376, Ep.Steps: 110, Score: -110.12, Avg.Score: -101.72, Time: 00:08:16\n",
      "Ep.: 20, Total Steps: 10976, Ep.Steps: 1600, Score: -81.26, Avg.Score: -100.74, Time: 00:09:47\n",
      "Ep.: 21, Total Steps: 11058, Ep.Steps: 82, Score: -107.29, Avg.Score: -101.04, Time: 00:09:52\n",
      "Ep.: 22, Total Steps: 11170, Ep.Steps: 112, Score: -125.14, Avg.Score: -102.09, Time: 00:09:58\n",
      "Ep.: 23, Total Steps: 11270, Ep.Steps: 100, Score: -127.84, Avg.Score: -103.16, Time: 00:10:04\n",
      "Ep.: 24, Total Steps: 11379, Ep.Steps: 109, Score: -102.96, Avg.Score: -103.15, Time: 00:10:11\n",
      "Ep.: 25, Total Steps: 11471, Ep.Steps: 92, Score: -107.43, Avg.Score: -103.32, Time: 00:10:16\n",
      "Ep.: 26, Total Steps: 11580, Ep.Steps: 109, Score: -125.46, Avg.Score: -104.14, Time: 00:10:22\n",
      "Ep.: 27, Total Steps: 11656, Ep.Steps: 76, Score: -108.95, Avg.Score: -104.31, Time: 00:10:27\n",
      "Ep.: 28, Total Steps: 11727, Ep.Steps: 71, Score: -106.26, Avg.Score: -104.38, Time: 00:10:31\n",
      "Ep.: 29, Total Steps: 11804, Ep.Steps: 77, Score: -100.74, Avg.Score: -104.26, Time: 00:10:35\n",
      "Ep.: 30, Total Steps: 11879, Ep.Steps: 75, Score: -104.35, Avg.Score: -104.26, Time: 00:10:40\n",
      "Ep.: 31, Total Steps: 11965, Ep.Steps: 86, Score: -104.77, Avg.Score: -104.27, Time: 00:10:45\n",
      "Ep.: 32, Total Steps: 12062, Ep.Steps: 97, Score: -103.02, Avg.Score: -104.24, Time: 00:10:50\n",
      "Ep.: 33, Total Steps: 12151, Ep.Steps: 89, Score: -106.90, Avg.Score: -104.32, Time: 00:10:56\n",
      "Ep.: 34, Total Steps: 12241, Ep.Steps: 90, Score: -101.21, Avg.Score: -104.23, Time: 00:11:01\n",
      "Ep.: 35, Total Steps: 12356, Ep.Steps: 115, Score: -128.16, Avg.Score: -104.89, Time: 00:11:07\n",
      "Ep.: 36, Total Steps: 12549, Ep.Steps: 193, Score: -102.24, Avg.Score: -104.82, Time: 00:11:19\n",
      "Ep.: 37, Total Steps: 12616, Ep.Steps: 67, Score: -124.13, Avg.Score: -105.33, Time: 00:11:23\n",
      "Ep.: 38, Total Steps: 12824, Ep.Steps: 208, Score: -134.26, Avg.Score: -106.07, Time: 00:11:35\n",
      "Ep.: 39, Total Steps: 13068, Ep.Steps: 244, Score: -133.06, Avg.Score: -106.74, Time: 00:11:49\n",
      "Ep.: 40, Total Steps: 13159, Ep.Steps: 91, Score: -119.33, Avg.Score: -107.05, Time: 00:11:54\n",
      "Ep.: 41, Total Steps: 13233, Ep.Steps: 74, Score: -101.89, Avg.Score: -106.93, Time: 00:11:59\n",
      "Ep.: 42, Total Steps: 13361, Ep.Steps: 128, Score: -133.95, Avg.Score: -107.56, Time: 00:12:06\n",
      "Ep.: 43, Total Steps: 14961, Ep.Steps: 1600, Score: -99.54, Avg.Score: -107.37, Time: 00:13:39\n",
      "Ep.: 44, Total Steps: 15032, Ep.Steps: 71, Score: -114.61, Avg.Score: -107.54, Time: 00:13:43\n",
      "Ep.: 45, Total Steps: 16632, Ep.Steps: 1600, Score: -114.14, Avg.Score: -107.68, Time: 00:15:16\n",
      "Ep.: 46, Total Steps: 16691, Ep.Steps: 59, Score: -110.99, Avg.Score: -107.75, Time: 00:15:19\n",
      "Ep.: 47, Total Steps: 16794, Ep.Steps: 103, Score: -107.92, Avg.Score: -107.75, Time: 00:15:25\n",
      "Ep.: 48, Total Steps: 16885, Ep.Steps: 91, Score: -115.82, Avg.Score: -107.92, Time: 00:15:31\n",
      "Ep.: 49, Total Steps: 16962, Ep.Steps: 77, Score: -116.06, Avg.Score: -108.08, Time: 00:15:35\n",
      "Ep.: 50, Total Steps: 17049, Ep.Steps: 87, Score: -116.91, Avg.Score: -108.25, Time: 00:15:41\n",
      "Ep.: 51, Total Steps: 17107, Ep.Steps: 58, Score: -113.29, Avg.Score: -108.35, Time: 00:15:44\n",
      "Ep.: 52, Total Steps: 17165, Ep.Steps: 58, Score: -116.71, Avg.Score: -108.51, Time: 00:15:47\n",
      "Ep.: 53, Total Steps: 17286, Ep.Steps: 121, Score: -117.31, Avg.Score: -108.67, Time: 00:15:54\n",
      "Ep.: 54, Total Steps: 17343, Ep.Steps: 57, Score: -116.34, Avg.Score: -108.81, Time: 00:15:58\n",
      "Ep.: 55, Total Steps: 17434, Ep.Steps: 91, Score: -128.73, Avg.Score: -109.17, Time: 00:16:03\n",
      "Ep.: 56, Total Steps: 17524, Ep.Steps: 90, Score: -124.76, Avg.Score: -109.44, Time: 00:16:08\n",
      "Ep.: 57, Total Steps: 19124, Ep.Steps: 1600, Score: -81.65, Avg.Score: -108.96, Time: 00:17:42\n",
      "Ep.: 58, Total Steps: 19224, Ep.Steps: 100, Score: -124.50, Avg.Score: -109.22, Time: 00:17:47\n",
      "Ep.: 59, Total Steps: 20824, Ep.Steps: 1600, Score: -97.64, Avg.Score: -109.03, Time: 00:19:20\n",
      "Ep.: 60, Total Steps: 22424, Ep.Steps: 1600, Score: -82.91, Avg.Score: -108.60, Time: 00:20:53\n",
      "Ep.: 61, Total Steps: 22586, Ep.Steps: 162, Score: -113.57, Avg.Score: -108.68, Time: 00:21:03\n",
      "Ep.: 62, Total Steps: 24186, Ep.Steps: 1600, Score: -87.74, Avg.Score: -108.35, Time: 00:22:36\n",
      "Ep.: 63, Total Steps: 24264, Ep.Steps: 78, Score: -119.58, Avg.Score: -108.53, Time: 00:22:41\n",
      "Ep.: 64, Total Steps: 25864, Ep.Steps: 1600, Score: -78.10, Avg.Score: -108.06, Time: 00:24:14\n",
      "Ep.: 65, Total Steps: 27464, Ep.Steps: 1600, Score: -73.79, Avg.Score: -107.54, Time: 00:25:46\n",
      "Ep.: 66, Total Steps: 29064, Ep.Steps: 1600, Score: -77.13, Avg.Score: -107.08, Time: 00:27:19\n",
      "Ep.: 67, Total Steps: 30664, Ep.Steps: 1600, Score: -86.99, Avg.Score: -106.79, Time: 00:28:53\n",
      "Ep.: 68, Total Steps: 30782, Ep.Steps: 118, Score: -112.30, Avg.Score: -106.87, Time: 00:28:59\n",
      "Ep.: 69, Total Steps: 30836, Ep.Steps: 54, Score: -110.44, Avg.Score: -106.92, Time: 00:29:03\n",
      "Ep.: 70, Total Steps: 32436, Ep.Steps: 1600, Score: -99.14, Avg.Score: -106.81, Time: 00:30:35\n",
      "Ep.: 71, Total Steps: 32544, Ep.Steps: 108, Score: -126.33, Avg.Score: -107.08, Time: 00:30:42\n",
      "Ep.: 72, Total Steps: 34144, Ep.Steps: 1600, Score: -92.95, Avg.Score: -106.89, Time: 00:32:15\n",
      "Ep.: 73, Total Steps: 35744, Ep.Steps: 1600, Score: -99.67, Avg.Score: -106.79, Time: 00:33:48\n",
      "Ep.: 74, Total Steps: 37344, Ep.Steps: 1600, Score: -77.74, Avg.Score: -106.40, Time: 00:35:21\n",
      "Ep.: 75, Total Steps: 38944, Ep.Steps: 1600, Score: -91.85, Avg.Score: -106.21, Time: 00:36:54\n",
      "Ep.: 76, Total Steps: 40544, Ep.Steps: 1600, Score: -97.87, Avg.Score: -106.10, Time: 00:38:28\n",
      "Ep.: 77, Total Steps: 42144, Ep.Steps: 1600, Score: -98.11, Avg.Score: -106.00, Time: 00:40:01\n",
      "Ep.: 78, Total Steps: 43744, Ep.Steps: 1600, Score: -86.24, Avg.Score: -105.75, Time: 00:41:34\n",
      "Ep.: 79, Total Steps: 45344, Ep.Steps: 1600, Score: -76.32, Avg.Score: -105.38, Time: 00:43:06\n",
      "Ep.: 80, Total Steps: 45900, Ep.Steps: 556, Score: -151.32, Avg.Score: -105.95, Time: 00:43:39\n",
      "Ep.: 81, Total Steps: 46011, Ep.Steps: 111, Score: -129.25, Avg.Score: -106.23, Time: 00:43:45\n",
      "Ep.: 82, Total Steps: 47611, Ep.Steps: 1600, Score: -114.98, Avg.Score: -106.34, Time: 00:45:18\n",
      "Ep.: 83, Total Steps: 47781, Ep.Steps: 170, Score: -138.21, Avg.Score: -106.72, Time: 00:45:28\n",
      "Ep.: 84, Total Steps: 47874, Ep.Steps: 93, Score: -130.48, Avg.Score: -107.00, Time: 00:45:33\n",
      "Ep.: 85, Total Steps: 49474, Ep.Steps: 1600, Score: -114.21, Avg.Score: -107.08, Time: 00:47:06\n",
      "Ep.: 86, Total Steps: 51074, Ep.Steps: 1600, Score: -104.15, Avg.Score: -107.05, Time: 00:48:41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 87, Total Steps: 52674, Ep.Steps: 1600, Score: -61.00, Avg.Score: -106.53, Time: 00:50:16\n",
      "Ep.: 88, Total Steps: 54274, Ep.Steps: 1600, Score: -72.29, Avg.Score: -106.14, Time: 00:51:49\n",
      "Ep.: 89, Total Steps: 55874, Ep.Steps: 1600, Score: -82.94, Avg.Score: -105.88, Time: 00:53:22\n",
      "Ep.: 90, Total Steps: 57474, Ep.Steps: 1600, Score: -76.12, Avg.Score: -105.56, Time: 00:54:55\n",
      "Ep.: 91, Total Steps: 59074, Ep.Steps: 1600, Score: -82.73, Avg.Score: -105.31, Time: 00:56:28\n",
      "Ep.: 92, Total Steps: 60674, Ep.Steps: 1600, Score: -110.70, Avg.Score: -105.37, Time: 00:58:01\n",
      "Ep.: 93, Total Steps: 62274, Ep.Steps: 1600, Score: -70.85, Avg.Score: -105.00, Time: 00:59:34\n",
      "Ep.: 94, Total Steps: 63874, Ep.Steps: 1600, Score: -78.17, Avg.Score: -104.72, Time: 01:01:07\n",
      "Ep.: 95, Total Steps: 63947, Ep.Steps: 73, Score: -121.28, Avg.Score: -104.89, Time: 01:01:11\n",
      "Ep.: 96, Total Steps: 65547, Ep.Steps: 1600, Score: -81.22, Avg.Score: -104.64, Time: 01:02:44\n",
      "Ep.: 97, Total Steps: 67147, Ep.Steps: 1600, Score: -88.13, Avg.Score: -104.48, Time: 01:04:17\n",
      "Ep.: 98, Total Steps: 67240, Ep.Steps: 93, Score: -123.54, Avg.Score: -104.67, Time: 01:04:23\n",
      "Ep.: 99, Total Steps: 68840, Ep.Steps: 1600, Score: -91.84, Avg.Score: -104.54, Time: 01:05:56\n",
      "Ep.: 100, Total Steps: 70440, Ep.Steps: 1600, Score: -89.40, Avg.Score: -104.27, Time: 01:07:29\n",
      "Ep.: 101, Total Steps: 72040, Ep.Steps: 1600, Score: -97.23, Avg.Score: -104.24, Time: 01:09:02\n",
      "Ep.: 102, Total Steps: 72111, Ep.Steps: 71, Score: -100.10, Avg.Score: -104.10, Time: 01:09:06\n",
      "Ep.: 103, Total Steps: 73711, Ep.Steps: 1600, Score: -121.49, Avg.Score: -104.45, Time: 01:10:39\n",
      "Ep.: 104, Total Steps: 75311, Ep.Steps: 1600, Score: -100.67, Avg.Score: -104.26, Time: 01:12:13\n",
      "Ep.: 105, Total Steps: 76911, Ep.Steps: 1600, Score: -80.09, Avg.Score: -104.06, Time: 01:13:46\n",
      "Ep.: 106, Total Steps: 78511, Ep.Steps: 1600, Score: -71.69, Avg.Score: -104.00, Time: 01:15:18\n",
      "Ep.: 107, Total Steps: 80111, Ep.Steps: 1600, Score: -60.60, Avg.Score: -103.53, Time: 01:16:51\n",
      "Ep.: 108, Total Steps: 81711, Ep.Steps: 1600, Score: -59.26, Avg.Score: -103.11, Time: 01:18:24\n",
      "Ep.: 109, Total Steps: 83277, Ep.Steps: 1566, Score: -122.66, Avg.Score: -103.32, Time: 01:19:55\n",
      "Ep.: 110, Total Steps: 84877, Ep.Steps: 1600, Score: -82.26, Avg.Score: -103.02, Time: 01:21:28\n",
      "Ep.: 111, Total Steps: 86477, Ep.Steps: 1600, Score: -37.56, Avg.Score: -102.40, Time: 01:23:00\n",
      "Ep.: 112, Total Steps: 88077, Ep.Steps: 1600, Score: -38.31, Avg.Score: -101.62, Time: 01:24:33\n",
      "Ep.: 113, Total Steps: 89677, Ep.Steps: 1600, Score: -34.26, Avg.Score: -100.95, Time: 01:26:07\n",
      "Ep.: 114, Total Steps: 91277, Ep.Steps: 1600, Score: -52.00, Avg.Score: -100.65, Time: 01:27:46\n",
      "Ep.: 115, Total Steps: 92877, Ep.Steps: 1600, Score: -68.49, Avg.Score: -100.45, Time: 01:29:21\n",
      "Ep.: 116, Total Steps: 92936, Ep.Steps: 59, Score: -106.67, Avg.Score: -100.73, Time: 01:29:25\n",
      "Ep.: 117, Total Steps: 94536, Ep.Steps: 1600, Score: -75.67, Avg.Score: -100.49, Time: 01:31:00\n",
      "Ep.: 118, Total Steps: 96136, Ep.Steps: 1600, Score: -63.15, Avg.Score: -99.91, Time: 01:32:36\n",
      "Ep.: 119, Total Steps: 97736, Ep.Steps: 1600, Score: -48.62, Avg.Score: -99.30, Time: 01:34:12\n",
      "Ep.: 120, Total Steps: 99336, Ep.Steps: 1600, Score: -54.33, Avg.Score: -99.03, Time: 01:35:46\n",
      "Ep.: 121, Total Steps: 100936, Ep.Steps: 1600, Score: -56.29, Avg.Score: -98.52, Time: 01:37:19\n",
      "Ep.: 122, Total Steps: 102536, Ep.Steps: 1600, Score: -67.52, Avg.Score: -97.94, Time: 01:38:53\n",
      "Ep.: 123, Total Steps: 104136, Ep.Steps: 1600, Score: -54.96, Avg.Score: -97.21, Time: 01:40:27\n",
      "Ep.: 124, Total Steps: 105736, Ep.Steps: 1600, Score: -52.18, Avg.Score: -96.71, Time: 01:42:00\n",
      "Ep.: 125, Total Steps: 107336, Ep.Steps: 1600, Score: -66.58, Avg.Score: -96.30, Time: 01:43:34\n",
      "Ep.: 126, Total Steps: 108936, Ep.Steps: 1600, Score: -74.86, Avg.Score: -95.79, Time: 01:45:08\n",
      "Ep.: 127, Total Steps: 110536, Ep.Steps: 1600, Score: -82.16, Avg.Score: -95.52, Time: 01:46:41\n",
      "Ep.: 128, Total Steps: 112136, Ep.Steps: 1600, Score: -72.82, Avg.Score: -95.19, Time: 01:48:15\n",
      "Ep.: 129, Total Steps: 113736, Ep.Steps: 1600, Score: -73.45, Avg.Score: -94.92, Time: 01:49:48\n",
      "Ep.: 130, Total Steps: 115336, Ep.Steps: 1600, Score: -71.76, Avg.Score: -94.59, Time: 01:51:22\n",
      "Ep.: 131, Total Steps: 116936, Ep.Steps: 1600, Score: -52.09, Avg.Score: -94.06, Time: 01:52:55\n",
      "Ep.: 132, Total Steps: 118490, Ep.Steps: 1554, Score: -117.62, Avg.Score: -94.21, Time: 01:54:27\n",
      "Ep.: 133, Total Steps: 120090, Ep.Steps: 1600, Score: -27.95, Avg.Score: -93.42, Time: 01:56:01\n",
      "Ep.: 134, Total Steps: 121690, Ep.Steps: 1600, Score: -42.82, Avg.Score: -92.84, Time: 01:57:34\n",
      "Ep.: 135, Total Steps: 123290, Ep.Steps: 1600, Score: -44.00, Avg.Score: -92.00, Time: 01:59:08\n",
      "Ep.: 136, Total Steps: 124890, Ep.Steps: 1600, Score: -30.98, Avg.Score: -91.28, Time: 02:00:48\n",
      "Ep.: 137, Total Steps: 126490, Ep.Steps: 1600, Score: -41.48, Avg.Score: -90.46, Time: 02:02:29\n",
      "Ep.: 138, Total Steps: 126590, Ep.Steps: 100, Score: -111.34, Avg.Score: -90.23, Time: 02:02:35\n",
      "Ep.: 139, Total Steps: 126681, Ep.Steps: 91, Score: -98.49, Avg.Score: -89.88, Time: 02:02:41\n",
      "Ep.: 140, Total Steps: 128281, Ep.Steps: 1600, Score: -48.84, Avg.Score: -89.18, Time: 02:04:37\n",
      "Ep.: 141, Total Steps: 129881, Ep.Steps: 1600, Score: -30.03, Avg.Score: -88.46, Time: 02:06:23\n",
      "Ep.: 142, Total Steps: 131481, Ep.Steps: 1600, Score: -31.56, Avg.Score: -87.43, Time: 02:08:09\n",
      "Ep.: 143, Total Steps: 133081, Ep.Steps: 1600, Score: -37.40, Avg.Score: -86.81, Time: 02:09:52\n",
      "Ep.: 144, Total Steps: 134681, Ep.Steps: 1600, Score: -30.62, Avg.Score: -85.97, Time: 02:11:32\n",
      "Ep.: 145, Total Steps: 136281, Ep.Steps: 1600, Score: -36.76, Avg.Score: -85.20, Time: 02:13:11\n",
      "Ep.: 146, Total Steps: 137881, Ep.Steps: 1600, Score: -33.06, Avg.Score: -84.42, Time: 02:14:48\n",
      "Ep.: 147, Total Steps: 139481, Ep.Steps: 1600, Score: -38.58, Avg.Score: -83.73, Time: 02:16:29\n",
      "Ep.: 148, Total Steps: 141081, Ep.Steps: 1600, Score: -31.41, Avg.Score: -82.88, Time: 02:18:14\n",
      "Ep.: 149, Total Steps: 141350, Ep.Steps: 269, Score: -137.03, Avg.Score: -83.09, Time: 02:18:30\n",
      "Ep.: 150, Total Steps: 142950, Ep.Steps: 1600, Score: -44.29, Avg.Score: -82.37, Time: 02:20:05\n",
      "Ep.: 151, Total Steps: 144550, Ep.Steps: 1600, Score: -39.88, Avg.Score: -81.63, Time: 02:21:40\n",
      "Ep.: 152, Total Steps: 146150, Ep.Steps: 1600, Score: -39.02, Avg.Score: -80.85, Time: 02:23:15\n",
      "Ep.: 153, Total Steps: 147750, Ep.Steps: 1600, Score: -26.48, Avg.Score: -79.95, Time: 02:24:50\n",
      "Ep.: 154, Total Steps: 149350, Ep.Steps: 1600, Score: -28.54, Avg.Score: -79.07, Time: 02:26:24\n",
      "Ep.: 155, Total Steps: 150950, Ep.Steps: 1600, Score: -33.20, Avg.Score: -78.11, Time: 02:27:59\n",
      "Ep.: 156, Total Steps: 152550, Ep.Steps: 1600, Score: -39.51, Avg.Score: -77.26, Time: 02:29:34\n",
      "Ep.: 157, Total Steps: 154150, Ep.Steps: 1600, Score: -56.89, Avg.Score: -77.01, Time: 02:31:08\n",
      "Ep.: 158, Total Steps: 155750, Ep.Steps: 1600, Score: -84.73, Avg.Score: -76.62, Time: 02:32:43\n",
      "Ep.: 159, Total Steps: 157350, Ep.Steps: 1600, Score: -67.86, Avg.Score: -76.32, Time: 02:34:25\n",
      "Ep.: 160, Total Steps: 158950, Ep.Steps: 1600, Score: -42.18, Avg.Score: -75.91, Time: 02:36:00\n",
      "Ep.: 161, Total Steps: 160550, Ep.Steps: 1600, Score: -22.02, Avg.Score: -74.99, Time: 02:37:40\n",
      "Ep.: 162, Total Steps: 162150, Ep.Steps: 1600, Score: -38.88, Avg.Score: -74.51, Time: 02:39:17\n",
      "Ep.: 163, Total Steps: 163750, Ep.Steps: 1600, Score: -72.08, Avg.Score: -74.03, Time: 02:40:52\n",
      "Ep.: 164, Total Steps: 165350, Ep.Steps: 1600, Score: -49.15, Avg.Score: -73.74, Time: 02:42:28\n",
      "Ep.: 165, Total Steps: 165422, Ep.Steps: 72, Score: -103.41, Avg.Score: -74.04, Time: 02:42:32\n",
      "Ep.: 166, Total Steps: 167022, Ep.Steps: 1600, Score: -43.20, Avg.Score: -73.70, Time: 02:44:07\n",
      "Ep.: 167, Total Steps: 168622, Ep.Steps: 1600, Score: -31.73, Avg.Score: -73.15, Time: 02:45:42\n",
      "Ep.: 168, Total Steps: 170222, Ep.Steps: 1600, Score: -41.33, Avg.Score: -72.44, Time: 02:47:17\n",
      "Ep.: 169, Total Steps: 171822, Ep.Steps: 1600, Score: -54.75, Avg.Score: -71.88, Time: 02:48:52\n",
      "Ep.: 170, Total Steps: 173422, Ep.Steps: 1600, Score: -32.48, Avg.Score: -71.21, Time: 02:50:26\n",
      "Ep.: 171, Total Steps: 175022, Ep.Steps: 1600, Score: -24.49, Avg.Score: -70.19, Time: 02:52:01\n",
      "Ep.: 172, Total Steps: 176622, Ep.Steps: 1600, Score: -6.29, Avg.Score: -69.33, Time: 02:53:35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 173, Total Steps: 178222, Ep.Steps: 1600, Score: -27.92, Avg.Score: -68.61, Time: 02:55:10\n",
      "Ep.: 174, Total Steps: 179822, Ep.Steps: 1600, Score: -13.59, Avg.Score: -67.97, Time: 02:56:44\n",
      "Ep.: 175, Total Steps: 181422, Ep.Steps: 1600, Score: -31.38, Avg.Score: -67.36, Time: 02:58:18\n",
      "Ep.: 176, Total Steps: 183022, Ep.Steps: 1600, Score: -23.95, Avg.Score: -66.63, Time: 02:59:52\n",
      "Ep.: 177, Total Steps: 184622, Ep.Steps: 1600, Score: -27.48, Avg.Score: -65.92, Time: 03:01:27\n",
      "Ep.: 178, Total Steps: 186222, Ep.Steps: 1600, Score: -39.70, Avg.Score: -65.45, Time: 03:03:01\n",
      "Ep.: 179, Total Steps: 187822, Ep.Steps: 1600, Score: -35.42, Avg.Score: -65.04, Time: 03:04:36\n",
      "Ep.: 180, Total Steps: 189422, Ep.Steps: 1600, Score: -29.06, Avg.Score: -63.82, Time: 03:06:10\n",
      "Ep.: 181, Total Steps: 191022, Ep.Steps: 1600, Score: -34.21, Avg.Score: -62.87, Time: 03:07:44\n",
      "Ep.: 182, Total Steps: 192622, Ep.Steps: 1600, Score: -23.99, Avg.Score: -61.96, Time: 03:09:18\n",
      "Ep.: 183, Total Steps: 194222, Ep.Steps: 1600, Score: -27.58, Avg.Score: -60.86, Time: 03:10:53\n",
      "Ep.: 184, Total Steps: 195822, Ep.Steps: 1600, Score: -25.04, Avg.Score: -59.80, Time: 03:12:27\n",
      "Ep.: 185, Total Steps: 197422, Ep.Steps: 1600, Score: -16.21, Avg.Score: -58.82, Time: 03:14:01\n",
      "Ep.: 186, Total Steps: 199022, Ep.Steps: 1600, Score: 8.24, Avg.Score: -57.70, Time: 03:15:36\n",
      "Ep.: 187, Total Steps: 200622, Ep.Steps: 1600, Score: 4.80, Avg.Score: -57.04, Time: 03:17:10\n",
      "Ep.: 188, Total Steps: 202222, Ep.Steps: 1600, Score: 11.68, Avg.Score: -56.20, Time: 03:18:45\n",
      "Ep.: 189, Total Steps: 203822, Ep.Steps: 1600, Score: 6.12, Avg.Score: -55.31, Time: 03:20:19\n",
      "Ep.: 190, Total Steps: 204034, Ep.Steps: 212, Score: -107.92, Avg.Score: -55.63, Time: 03:20:31\n",
      "Ep.: 191, Total Steps: 205634, Ep.Steps: 1600, Score: 11.68, Avg.Score: -54.68, Time: 03:22:05\n",
      "Ep.: 192, Total Steps: 207234, Ep.Steps: 1600, Score: 8.59, Avg.Score: -53.49, Time: 03:23:40\n",
      "Ep.: 193, Total Steps: 208834, Ep.Steps: 1600, Score: 4.09, Avg.Score: -52.74, Time: 03:25:14\n",
      "Ep.: 194, Total Steps: 210434, Ep.Steps: 1600, Score: 13.42, Avg.Score: -51.82, Time: 03:26:48\n",
      "Ep.: 195, Total Steps: 212034, Ep.Steps: 1600, Score: 15.11, Avg.Score: -50.46, Time: 03:28:23\n",
      "Ep.: 196, Total Steps: 213634, Ep.Steps: 1600, Score: 12.37, Avg.Score: -49.52, Time: 03:29:58\n",
      "Ep.: 197, Total Steps: 215234, Ep.Steps: 1600, Score: 19.97, Avg.Score: -48.44, Time: 03:31:32\n",
      "Ep.: 198, Total Steps: 216834, Ep.Steps: 1600, Score: 20.18, Avg.Score: -47.01, Time: 03:33:06\n",
      "Ep.: 199, Total Steps: 218434, Ep.Steps: 1600, Score: 2.67, Avg.Score: -46.06, Time: 03:34:41\n",
      "Ep.: 200, Total Steps: 220034, Ep.Steps: 1600, Score: 2.47, Avg.Score: -45.14, Time: 03:36:15\n",
      "Ep.: 201, Total Steps: 221634, Ep.Steps: 1600, Score: -0.93, Avg.Score: -44.18, Time: 03:37:49\n",
      "Ep.: 202, Total Steps: 223234, Ep.Steps: 1600, Score: -4.20, Avg.Score: -43.22, Time: 03:39:24\n",
      "Ep.: 203, Total Steps: 224834, Ep.Steps: 1600, Score: -3.74, Avg.Score: -42.04, Time: 03:40:58\n",
      "Ep.: 204, Total Steps: 226434, Ep.Steps: 1600, Score: -8.20, Avg.Score: -41.12, Time: 03:42:32\n",
      "Ep.: 205, Total Steps: 228034, Ep.Steps: 1600, Score: 8.77, Avg.Score: -40.23, Time: 03:44:07\n",
      "Ep.: 206, Total Steps: 229634, Ep.Steps: 1600, Score: -2.96, Avg.Score: -39.54, Time: 03:45:42\n",
      "Ep.: 207, Total Steps: 231234, Ep.Steps: 1600, Score: -6.78, Avg.Score: -39.00, Time: 03:47:16\n",
      "Ep.: 208, Total Steps: 232834, Ep.Steps: 1600, Score: 0.07, Avg.Score: -38.41, Time: 03:48:51\n",
      "Ep.: 209, Total Steps: 234434, Ep.Steps: 1600, Score: 6.46, Avg.Score: -37.12, Time: 03:50:25\n",
      "Ep.: 210, Total Steps: 236034, Ep.Steps: 1600, Score: 4.56, Avg.Score: -36.25, Time: 03:52:00\n",
      "Ep.: 211, Total Steps: 237634, Ep.Steps: 1600, Score: 3.39, Avg.Score: -35.84, Time: 03:53:35\n",
      "Ep.: 212, Total Steps: 239234, Ep.Steps: 1600, Score: -17.01, Avg.Score: -35.63, Time: 03:55:09\n",
      "Ep.: 213, Total Steps: 240834, Ep.Steps: 1600, Score: -14.32, Avg.Score: -35.43, Time: 03:56:43\n",
      "Ep.: 214, Total Steps: 242434, Ep.Steps: 1600, Score: 27.35, Avg.Score: -34.64, Time: 03:58:18\n",
      "Ep.: 215, Total Steps: 244034, Ep.Steps: 1600, Score: 33.12, Avg.Score: -33.62, Time: 03:59:52\n",
      "Ep.: 216, Total Steps: 245634, Ep.Steps: 1600, Score: 30.24, Avg.Score: -32.25, Time: 04:01:27\n",
      "Ep.: 217, Total Steps: 247234, Ep.Steps: 1600, Score: 30.34, Avg.Score: -31.19, Time: 04:03:01\n",
      "Ep.: 218, Total Steps: 248834, Ep.Steps: 1600, Score: 8.94, Avg.Score: -30.47, Time: 04:04:36\n",
      "Ep.: 219, Total Steps: 250434, Ep.Steps: 1600, Score: 39.82, Avg.Score: -29.58, Time: 04:06:10\n",
      "Ep.: 220, Total Steps: 252034, Ep.Steps: 1600, Score: 43.24, Avg.Score: -28.61, Time: 04:07:45\n",
      "Ep.: 221, Total Steps: 253634, Ep.Steps: 1600, Score: 38.48, Avg.Score: -27.66, Time: 04:09:19\n",
      "Ep.: 222, Total Steps: 255234, Ep.Steps: 1600, Score: -15.56, Avg.Score: -27.14, Time: 04:10:54\n",
      "Ep.: 223, Total Steps: 256834, Ep.Steps: 1600, Score: -2.14, Avg.Score: -26.61, Time: 04:12:28\n",
      "Ep.: 224, Total Steps: 258434, Ep.Steps: 1600, Score: -3.24, Avg.Score: -26.12, Time: 04:14:02\n",
      "Ep.: 225, Total Steps: 258525, Ep.Steps: 91, Score: -100.29, Avg.Score: -26.46, Time: 04:14:08\n",
      "Ep.: 226, Total Steps: 260125, Ep.Steps: 1600, Score: 38.91, Avg.Score: -25.32, Time: 04:15:42\n",
      "Ep.: 227, Total Steps: 260215, Ep.Steps: 90, Score: -108.38, Avg.Score: -25.59, Time: 04:15:48\n",
      "Ep.: 228, Total Steps: 260288, Ep.Steps: 73, Score: -104.86, Avg.Score: -25.91, Time: 04:15:52\n",
      "Ep.: 229, Total Steps: 260343, Ep.Steps: 55, Score: -105.20, Avg.Score: -26.22, Time: 04:15:55\n",
      "Ep.: 230, Total Steps: 260399, Ep.Steps: 56, Score: -106.97, Avg.Score: -26.58, Time: 04:15:58\n",
      "Ep.: 231, Total Steps: 260458, Ep.Steps: 59, Score: -111.04, Avg.Score: -27.17, Time: 04:16:02\n",
      "Ep.: 232, Total Steps: 260697, Ep.Steps: 239, Score: -96.16, Avg.Score: -26.95, Time: 04:16:16\n",
      "Ep.: 233, Total Steps: 262297, Ep.Steps: 1600, Score: 28.92, Avg.Score: -26.38, Time: 04:17:50\n",
      "Ep.: 234, Total Steps: 262382, Ep.Steps: 85, Score: -100.29, Avg.Score: -26.96, Time: 04:17:55\n",
      "Ep.: 235, Total Steps: 262542, Ep.Steps: 160, Score: -144.92, Avg.Score: -27.97, Time: 04:18:05\n",
      "Ep.: 236, Total Steps: 264142, Ep.Steps: 1600, Score: 20.67, Avg.Score: -27.45, Time: 04:19:39\n",
      "Ep.: 237, Total Steps: 265742, Ep.Steps: 1600, Score: 46.02, Avg.Score: -26.57, Time: 04:21:13\n",
      "Ep.: 238, Total Steps: 267342, Ep.Steps: 1600, Score: 14.05, Avg.Score: -25.32, Time: 04:22:47\n",
      "Ep.: 239, Total Steps: 268942, Ep.Steps: 1600, Score: 16.70, Avg.Score: -24.17, Time: 04:24:22\n",
      "Ep.: 240, Total Steps: 270542, Ep.Steps: 1600, Score: 54.55, Avg.Score: -23.13, Time: 04:25:56\n",
      "Ep.: 241, Total Steps: 272142, Ep.Steps: 1600, Score: 5.39, Avg.Score: -22.78, Time: 04:27:31\n",
      "Ep.: 242, Total Steps: 273742, Ep.Steps: 1600, Score: 1.57, Avg.Score: -22.45, Time: 04:29:05\n",
      "Ep.: 243, Total Steps: 275342, Ep.Steps: 1600, Score: 21.33, Avg.Score: -21.86, Time: 04:30:39\n",
      "Ep.: 244, Total Steps: 276942, Ep.Steps: 1600, Score: 15.25, Avg.Score: -21.40, Time: 04:32:28\n",
      "Ep.: 245, Total Steps: 278542, Ep.Steps: 1600, Score: 0.97, Avg.Score: -21.03, Time: 04:34:17\n",
      "Ep.: 246, Total Steps: 280142, Ep.Steps: 1600, Score: -5.13, Avg.Score: -20.75, Time: 04:35:52\n",
      "Ep.: 247, Total Steps: 281742, Ep.Steps: 1600, Score: -10.96, Avg.Score: -20.47, Time: 04:37:27\n",
      "Ep.: 248, Total Steps: 283342, Ep.Steps: 1600, Score: 63.83, Avg.Score: -19.52, Time: 04:39:02\n",
      "Ep.: 249, Total Steps: 284942, Ep.Steps: 1600, Score: 45.70, Avg.Score: -17.69, Time: 04:40:37\n",
      "Ep.: 250, Total Steps: 285067, Ep.Steps: 125, Score: -95.37, Avg.Score: -18.20, Time: 04:40:45\n",
      "Ep.: 251, Total Steps: 286667, Ep.Steps: 1600, Score: 51.15, Avg.Score: -17.29, Time: 04:42:21\n",
      "Ep.: 252, Total Steps: 286835, Ep.Steps: 168, Score: -99.21, Avg.Score: -17.89, Time: 04:42:31\n",
      "Ep.: 253, Total Steps: 288435, Ep.Steps: 1600, Score: 56.68, Avg.Score: -17.06, Time: 04:44:09\n",
      "Ep.: 254, Total Steps: 290035, Ep.Steps: 1600, Score: 66.26, Avg.Score: -16.11, Time: 04:45:50\n",
      "Ep.: 255, Total Steps: 290674, Ep.Steps: 639, Score: -72.50, Avg.Score: -16.51, Time: 04:46:30\n",
      "Ep.: 256, Total Steps: 291157, Ep.Steps: 483, Score: -80.79, Avg.Score: -16.92, Time: 04:47:00\n",
      "Ep.: 257, Total Steps: 292757, Ep.Steps: 1600, Score: 75.02, Avg.Score: -15.60, Time: 04:48:34\n",
      "Ep.: 258, Total Steps: 293181, Ep.Steps: 424, Score: -89.02, Avg.Score: -15.64, Time: 04:48:59\n",
      "Ep.: 259, Total Steps: 293248, Ep.Steps: 67, Score: -124.42, Avg.Score: -16.21, Time: 04:49:03\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 260, Total Steps: 294848, Ep.Steps: 1600, Score: 62.66, Avg.Score: -15.16, Time: 04:50:38\n",
      "Ep.: 261, Total Steps: 296448, Ep.Steps: 1600, Score: 17.92, Avg.Score: -14.76, Time: 04:52:11\n",
      "Ep.: 262, Total Steps: 296547, Ep.Steps: 99, Score: -127.31, Avg.Score: -15.64, Time: 04:52:17\n",
      "Ep.: 263, Total Steps: 298147, Ep.Steps: 1600, Score: 55.96, Avg.Score: -14.36, Time: 04:53:50\n",
      "Ep.: 264, Total Steps: 298198, Ep.Steps: 51, Score: -117.85, Avg.Score: -15.05, Time: 04:53:52\n",
      "Ep.: 265, Total Steps: 298251, Ep.Steps: 53, Score: -116.55, Avg.Score: -15.18, Time: 04:53:56\n",
      "Ep.: 266, Total Steps: 298299, Ep.Steps: 48, Score: -112.15, Avg.Score: -15.87, Time: 04:53:58\n",
      "Ep.: 267, Total Steps: 298363, Ep.Steps: 64, Score: -116.55, Avg.Score: -16.72, Time: 04:54:02\n",
      "Ep.: 268, Total Steps: 298408, Ep.Steps: 45, Score: -111.85, Avg.Score: -17.43, Time: 04:54:05\n",
      "Ep.: 269, Total Steps: 298468, Ep.Steps: 60, Score: -120.20, Avg.Score: -18.08, Time: 04:54:08\n",
      "Ep.: 270, Total Steps: 298561, Ep.Steps: 93, Score: -125.59, Avg.Score: -19.01, Time: 04:54:14\n",
      "Ep.: 271, Total Steps: 298639, Ep.Steps: 78, Score: -118.30, Avg.Score: -19.95, Time: 04:54:18\n",
      "Ep.: 272, Total Steps: 298689, Ep.Steps: 50, Score: -114.94, Avg.Score: -21.04, Time: 04:54:21\n",
      "Ep.: 273, Total Steps: 300289, Ep.Steps: 1600, Score: 56.21, Avg.Score: -20.19, Time: 04:55:55\n",
      "Ep.: 274, Total Steps: 301889, Ep.Steps: 1600, Score: 87.32, Avg.Score: -19.19, Time: 04:57:30\n",
      "Ep.: 275, Total Steps: 302012, Ep.Steps: 123, Score: -123.50, Avg.Score: -20.11, Time: 04:57:37\n",
      "Ep.: 276, Total Steps: 303612, Ep.Steps: 1600, Score: -15.82, Avg.Score: -20.03, Time: 04:59:32\n",
      "Ep.: 277, Total Steps: 305212, Ep.Steps: 1600, Score: 67.53, Avg.Score: -19.08, Time: 05:01:13\n",
      "Ep.: 278, Total Steps: 306812, Ep.Steps: 1600, Score: 66.83, Avg.Score: -18.01, Time: 05:02:56\n",
      "Ep.: 279, Total Steps: 308412, Ep.Steps: 1600, Score: 75.63, Avg.Score: -16.90, Time: 05:04:31\n",
      "Ep.: 280, Total Steps: 310012, Ep.Steps: 1600, Score: 92.58, Avg.Score: -15.68, Time: 05:06:05\n",
      "Ep.: 281, Total Steps: 311612, Ep.Steps: 1600, Score: 82.43, Avg.Score: -14.52, Time: 05:07:38\n",
      "Ep.: 282, Total Steps: 313212, Ep.Steps: 1600, Score: 82.08, Avg.Score: -13.46, Time: 05:09:11\n",
      "Ep.: 283, Total Steps: 314812, Ep.Steps: 1600, Score: 78.43, Avg.Score: -12.40, Time: 05:10:45\n",
      "Ep.: 284, Total Steps: 316412, Ep.Steps: 1600, Score: 6.38, Avg.Score: -12.08, Time: 05:12:18\n",
      "Ep.: 285, Total Steps: 318012, Ep.Steps: 1600, Score: 10.07, Avg.Score: -11.82, Time: 05:13:52\n",
      "Ep.: 286, Total Steps: 319612, Ep.Steps: 1600, Score: 13.51, Avg.Score: -11.77, Time: 05:15:27\n",
      "Ep.: 287, Total Steps: 321212, Ep.Steps: 1600, Score: 13.57, Avg.Score: -11.68, Time: 05:17:06\n",
      "Ep.: 288, Total Steps: 322812, Ep.Steps: 1600, Score: 86.02, Avg.Score: -10.94, Time: 05:18:45\n",
      "Ep.: 289, Total Steps: 324412, Ep.Steps: 1600, Score: 97.89, Avg.Score: -10.02, Time: 05:20:20\n",
      "Ep.: 290, Total Steps: 326012, Ep.Steps: 1600, Score: 12.00, Avg.Score: -8.82, Time: 05:21:53\n",
      "Ep.: 291, Total Steps: 327612, Ep.Steps: 1600, Score: 60.21, Avg.Score: -8.33, Time: 05:23:29\n",
      "Ep.: 292, Total Steps: 329212, Ep.Steps: 1600, Score: 92.50, Avg.Score: -7.49, Time: 05:25:03\n",
      "Ep.: 293, Total Steps: 330812, Ep.Steps: 1600, Score: 126.02, Avg.Score: -6.27, Time: 05:26:37\n",
      "Ep.: 294, Total Steps: 332412, Ep.Steps: 1600, Score: 135.78, Avg.Score: -5.05, Time: 05:28:10\n",
      "Ep.: 295, Total Steps: 334012, Ep.Steps: 1600, Score: 152.18, Avg.Score: -3.68, Time: 05:29:42\n",
      "Ep.: 296, Total Steps: 335612, Ep.Steps: 1600, Score: 143.76, Avg.Score: -2.37, Time: 05:31:15\n",
      "Ep.: 297, Total Steps: 337212, Ep.Steps: 1600, Score: 134.31, Avg.Score: -1.22, Time: 05:32:48\n",
      "Ep.: 298, Total Steps: 338812, Ep.Steps: 1600, Score: 137.43, Avg.Score: -0.05, Time: 05:34:21\n",
      "Ep.: 299, Total Steps: 340412, Ep.Steps: 1600, Score: 160.27, Avg.Score: 1.53, Time: 05:35:54\n",
      "Ep.: 300, Total Steps: 341645, Ep.Steps: 1233, Score: 26.39, Avg.Score: 1.76, Time: 05:37:05\n",
      "Ep.: 301, Total Steps: 341918, Ep.Steps: 273, Score: -68.11, Avg.Score: 1.09, Time: 05:37:21\n",
      "Ep.: 302, Total Steps: 343518, Ep.Steps: 1600, Score: 167.87, Avg.Score: 2.81, Time: 05:38:54\n",
      "Ep.: 303, Total Steps: 345118, Ep.Steps: 1600, Score: 213.18, Avg.Score: 4.98, Time: 05:40:27\n",
      "Ep.: 304, Total Steps: 346718, Ep.Steps: 1600, Score: 208.07, Avg.Score: 7.15, Time: 05:42:01\n",
      "Ep.: 305, Total Steps: 348318, Ep.Steps: 1600, Score: 212.93, Avg.Score: 9.19, Time: 05:43:33\n",
      "Ep.: 306, Total Steps: 349918, Ep.Steps: 1600, Score: 233.94, Avg.Score: 11.56, Time: 05:45:06\n",
      "Ep.: 307, Total Steps: 351518, Ep.Steps: 1600, Score: 208.82, Avg.Score: 13.71, Time: 05:46:39\n",
      "Ep.: 308, Total Steps: 352793, Ep.Steps: 1275, Score: 53.85, Avg.Score: 14.25, Time: 05:47:53\n",
      "Ep.: 309, Total Steps: 354393, Ep.Steps: 1600, Score: 204.91, Avg.Score: 16.23, Time: 05:49:25\n",
      "Ep.: 310, Total Steps: 355993, Ep.Steps: 1600, Score: 218.26, Avg.Score: 18.37, Time: 05:50:58\n",
      "Ep.: 311, Total Steps: 356631, Ep.Steps: 638, Score: -31.65, Avg.Score: 18.02, Time: 05:51:36\n",
      "Ep.: 312, Total Steps: 358231, Ep.Steps: 1600, Score: 221.43, Avg.Score: 20.40, Time: 05:53:08\n",
      "Ep.: 313, Total Steps: 359831, Ep.Steps: 1600, Score: 220.54, Avg.Score: 22.75, Time: 05:54:41\n",
      "Ep.: 314, Total Steps: 361431, Ep.Steps: 1600, Score: 87.03, Avg.Score: 23.35, Time: 05:56:14\n",
      "Ep.: 315, Total Steps: 363031, Ep.Steps: 1600, Score: 234.77, Avg.Score: 25.37, Time: 05:57:47\n",
      "Ep.: 316, Total Steps: 364631, Ep.Steps: 1600, Score: 244.31, Avg.Score: 27.51, Time: 05:59:20\n",
      "Ep.: 317, Total Steps: 366231, Ep.Steps: 1600, Score: 248.49, Avg.Score: 29.69, Time: 06:00:59\n",
      "Ep.: 318, Total Steps: 366764, Ep.Steps: 533, Score: -37.09, Avg.Score: 29.23, Time: 06:01:31\n",
      "Ep.: 319, Total Steps: 368364, Ep.Steps: 1600, Score: 252.96, Avg.Score: 31.36, Time: 06:03:08\n",
      "Ep.: 320, Total Steps: 369920, Ep.Steps: 1556, Score: 253.14, Avg.Score: 33.46, Time: 06:04:40\n",
      "Ep.: 321, Total Steps: 371520, Ep.Steps: 1600, Score: 248.60, Avg.Score: 35.56, Time: 06:06:13\n",
      "Ep.: 322, Total Steps: 373100, Ep.Steps: 1580, Score: 252.72, Avg.Score: 38.24, Time: 06:07:45\n",
      "Ep.: 323, Total Steps: 374159, Ep.Steps: 1059, Score: 86.84, Avg.Score: 39.13, Time: 06:08:46\n",
      "Ep.: 324, Total Steps: 375721, Ep.Steps: 1562, Score: 259.94, Avg.Score: 41.76, Time: 06:10:17\n",
      "Ep.: 325, Total Steps: 377259, Ep.Steps: 1538, Score: 263.93, Avg.Score: 45.41, Time: 06:11:46\n",
      "Ep.: 326, Total Steps: 378780, Ep.Steps: 1521, Score: 260.89, Avg.Score: 47.63, Time: 06:13:15\n",
      "Ep.: 327, Total Steps: 379899, Ep.Steps: 1119, Score: 102.51, Avg.Score: 49.74, Time: 06:14:20\n",
      "Ep.: 328, Total Steps: 380028, Ep.Steps: 129, Score: -79.95, Avg.Score: 49.98, Time: 06:14:27\n",
      "Ep.: 329, Total Steps: 380575, Ep.Steps: 547, Score: -47.07, Avg.Score: 50.57, Time: 06:14:59\n",
      "Ep.: 330, Total Steps: 380725, Ep.Steps: 150, Score: -85.16, Avg.Score: 50.78, Time: 06:15:07\n",
      "Ep.: 331, Total Steps: 380969, Ep.Steps: 244, Score: -68.59, Avg.Score: 51.21, Time: 06:15:22\n",
      "Ep.: 332, Total Steps: 381490, Ep.Steps: 521, Score: -14.69, Avg.Score: 52.02, Time: 06:15:52\n",
      "Ep.: 333, Total Steps: 381582, Ep.Steps: 92, Score: -99.19, Avg.Score: 50.74, Time: 06:15:57\n",
      "Ep.: 334, Total Steps: 381964, Ep.Steps: 382, Score: -37.10, Avg.Score: 51.37, Time: 06:16:19\n",
      "Ep.: 335, Total Steps: 382184, Ep.Steps: 220, Score: -61.69, Avg.Score: 52.21, Time: 06:16:32\n",
      "Ep.: 336, Total Steps: 382835, Ep.Steps: 651, Score: 10.13, Avg.Score: 52.10, Time: 06:17:10\n",
      "Ep.: 337, Total Steps: 383235, Ep.Steps: 400, Score: -25.00, Avg.Score: 51.39, Time: 06:17:33\n",
      "Ep.: 338, Total Steps: 383579, Ep.Steps: 344, Score: -45.81, Avg.Score: 50.79, Time: 06:17:53\n",
      "Ep.: 339, Total Steps: 384849, Ep.Steps: 1270, Score: 117.33, Avg.Score: 51.80, Time: 06:19:06\n",
      "Ep.: 340, Total Steps: 386315, Ep.Steps: 1466, Score: 262.20, Avg.Score: 53.88, Time: 06:20:31\n",
      "Ep.: 341, Total Steps: 387767, Ep.Steps: 1452, Score: 260.62, Avg.Score: 56.43, Time: 06:21:56\n",
      "Ep.: 342, Total Steps: 388013, Ep.Steps: 246, Score: -63.32, Avg.Score: 55.78, Time: 06:22:10\n",
      "Ep.: 343, Total Steps: 389531, Ep.Steps: 1518, Score: 255.69, Avg.Score: 58.12, Time: 06:23:38\n",
      "Ep.: 344, Total Steps: 391006, Ep.Steps: 1475, Score: 259.25, Avg.Score: 60.56, Time: 06:25:03\n",
      "Ep.: 345, Total Steps: 392569, Ep.Steps: 1563, Score: 246.08, Avg.Score: 63.01, Time: 06:26:34\n",
      "Ep.: 346, Total Steps: 393831, Ep.Steps: 1262, Score: 71.00, Avg.Score: 63.77, Time: 06:27:46\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 347, Total Steps: 394442, Ep.Steps: 611, Score: -38.32, Avg.Score: 63.50, Time: 06:28:22\n",
      "Ep.: 348, Total Steps: 396042, Ep.Steps: 1600, Score: 239.59, Avg.Score: 65.26, Time: 06:30:04\n",
      "Ep.: 349, Total Steps: 396113, Ep.Steps: 71, Score: -119.11, Avg.Score: 63.61, Time: 06:30:09\n",
      "Ep.: 350, Total Steps: 397066, Ep.Steps: 953, Score: -19.19, Avg.Score: 64.37, Time: 06:31:05\n",
      "Ep.: 351, Total Steps: 398666, Ep.Steps: 1600, Score: 226.97, Avg.Score: 66.13, Time: 06:32:38\n",
      "Ep.: 352, Total Steps: 400266, Ep.Steps: 1600, Score: 211.34, Avg.Score: 69.24, Time: 06:34:11\n",
      "Ep.: 353, Total Steps: 401748, Ep.Steps: 1482, Score: 118.34, Avg.Score: 69.85, Time: 06:35:38\n",
      "Ep.: 354, Total Steps: 403327, Ep.Steps: 1579, Score: 252.88, Avg.Score: 71.72, Time: 06:37:09\n",
      "Ep.: 355, Total Steps: 404824, Ep.Steps: 1497, Score: 259.85, Avg.Score: 75.04, Time: 06:38:36\n",
      "Ep.: 356, Total Steps: 406336, Ep.Steps: 1512, Score: 254.81, Avg.Score: 78.40, Time: 06:40:04\n",
      "Ep.: 357, Total Steps: 407780, Ep.Steps: 1444, Score: 262.26, Avg.Score: 80.27, Time: 06:41:27\n",
      "Ep.: 358, Total Steps: 409337, Ep.Steps: 1557, Score: 251.77, Avg.Score: 83.68, Time: 06:42:57\n",
      "Ep.: 359, Total Steps: 410821, Ep.Steps: 1484, Score: 256.94, Avg.Score: 87.49, Time: 06:44:23\n",
      "Ep.: 360, Total Steps: 412201, Ep.Steps: 1380, Score: 116.27, Avg.Score: 88.03, Time: 06:45:43\n",
      "Ep.: 361, Total Steps: 413732, Ep.Steps: 1531, Score: 254.52, Avg.Score: 90.39, Time: 06:47:12\n",
      "Ep.: 362, Total Steps: 415286, Ep.Steps: 1554, Score: 256.92, Avg.Score: 94.24, Time: 06:48:41\n",
      "Ep.: 363, Total Steps: 416780, Ep.Steps: 1494, Score: 262.77, Avg.Score: 96.30, Time: 06:50:08\n",
      "Ep.: 364, Total Steps: 418180, Ep.Steps: 1400, Score: 266.49, Avg.Score: 100.15, Time: 06:51:29\n",
      "Ep.: 365, Total Steps: 419780, Ep.Steps: 1600, Score: 241.14, Avg.Score: 103.72, Time: 06:53:02\n",
      "Ep.: 366, Total Steps: 421259, Ep.Steps: 1479, Score: 264.16, Avg.Score: 107.49, Time: 06:54:28\n",
      "Ep.: 367, Total Steps: 422839, Ep.Steps: 1580, Score: 252.63, Avg.Score: 111.18, Time: 06:55:59\n",
      "Ep.: 368, Total Steps: 424250, Ep.Steps: 1411, Score: 125.92, Avg.Score: 113.56, Time: 06:57:21\n",
      "Ep.: 369, Total Steps: 425752, Ep.Steps: 1502, Score: 260.63, Avg.Score: 117.37, Time: 06:58:48\n",
      "Ep.: 370, Total Steps: 427100, Ep.Steps: 1348, Score: 271.80, Avg.Score: 121.34, Time: 07:00:06\n",
      "Ep.: 371, Total Steps: 428544, Ep.Steps: 1444, Score: 266.23, Avg.Score: 125.18, Time: 07:01:30\n",
      "Ep.: 372, Total Steps: 430009, Ep.Steps: 1465, Score: 265.62, Avg.Score: 128.99, Time: 07:02:54\n",
      "Ep.: 373, Total Steps: 431590, Ep.Steps: 1581, Score: 259.52, Avg.Score: 131.02, Time: 07:04:26\n",
      "Ep.: 374, Total Steps: 433060, Ep.Steps: 1470, Score: 265.38, Avg.Score: 132.80, Time: 07:05:51\n",
      "Ep.: 375, Total Steps: 434569, Ep.Steps: 1509, Score: 263.98, Avg.Score: 136.68, Time: 07:07:19\n",
      "Ep.: 376, Total Steps: 436169, Ep.Steps: 1600, Score: 244.51, Avg.Score: 139.28, Time: 07:08:51\n",
      "Ep.: 377, Total Steps: 437667, Ep.Steps: 1498, Score: 265.87, Avg.Score: 141.27, Time: 07:10:18\n",
      "Ep.: 378, Total Steps: 439121, Ep.Steps: 1454, Score: 265.82, Avg.Score: 143.26, Time: 07:11:42\n",
      "Ep.: 379, Total Steps: 440435, Ep.Steps: 1314, Score: 272.58, Avg.Score: 145.22, Time: 07:12:58\n",
      "Ep.: 380, Total Steps: 441796, Ep.Steps: 1361, Score: 272.28, Avg.Score: 147.02, Time: 07:14:17\n",
      "Ep.: 381, Total Steps: 443105, Ep.Steps: 1309, Score: 276.23, Avg.Score: 148.96, Time: 07:15:33\n",
      "Ep.: 382, Total Steps: 444431, Ep.Steps: 1326, Score: 272.15, Avg.Score: 150.86, Time: 07:16:50\n",
      "Ep.: 383, Total Steps: 445733, Ep.Steps: 1302, Score: 274.11, Avg.Score: 152.82, Time: 07:18:05\n",
      "Ep.: 384, Total Steps: 447075, Ep.Steps: 1342, Score: 274.50, Avg.Score: 155.50, Time: 07:19:23\n",
      "Ep.: 385, Total Steps: 448378, Ep.Steps: 1303, Score: 277.24, Avg.Score: 158.17, Time: 07:20:38\n",
      "Ep.: 386, Total Steps: 449687, Ep.Steps: 1309, Score: 274.41, Avg.Score: 160.78, Time: 07:21:54\n",
      "Ep.: 387, Total Steps: 451013, Ep.Steps: 1326, Score: 270.92, Avg.Score: 163.35, Time: 07:23:11\n",
      "Ep.: 388, Total Steps: 452395, Ep.Steps: 1382, Score: 272.29, Avg.Score: 165.22, Time: 07:24:31\n",
      "Ep.: 389, Total Steps: 453691, Ep.Steps: 1296, Score: 274.47, Avg.Score: 166.98, Time: 07:25:46\n",
      "Ep.: 390, Total Steps: 455016, Ep.Steps: 1325, Score: 276.09, Avg.Score: 169.62, Time: 07:27:03\n",
      "Ep.: 391, Total Steps: 456473, Ep.Steps: 1457, Score: 262.29, Avg.Score: 171.64, Time: 07:28:27\n",
      "Ep.: 392, Total Steps: 457777, Ep.Steps: 1304, Score: 273.67, Avg.Score: 173.45, Time: 07:29:43\n",
      "Ep.: 393, Total Steps: 459235, Ep.Steps: 1458, Score: 264.76, Avg.Score: 174.84, Time: 07:31:07\n",
      "Ep.: 394, Total Steps: 460662, Ep.Steps: 1427, Score: 269.11, Avg.Score: 176.18, Time: 07:32:30\n",
      "Ep.: 395, Total Steps: 462017, Ep.Steps: 1355, Score: 269.82, Avg.Score: 177.35, Time: 07:33:48\n",
      "Ep.: 396, Total Steps: 463309, Ep.Steps: 1292, Score: 275.36, Avg.Score: 178.67, Time: 07:35:03\n",
      "Ep.: 397, Total Steps: 464604, Ep.Steps: 1295, Score: 271.00, Avg.Score: 180.03, Time: 07:36:18\n",
      "Ep.: 398, Total Steps: 465832, Ep.Steps: 1228, Score: 279.46, Avg.Score: 181.46, Time: 07:37:29\n",
      "Ep.: 399, Total Steps: 467161, Ep.Steps: 1329, Score: 274.14, Avg.Score: 182.59, Time: 07:38:46\n",
      "Ep.: 400, Total Steps: 468357, Ep.Steps: 1196, Score: 280.88, Avg.Score: 185.14, Time: 07:39:55\n",
      "Ep.: 401, Total Steps: 469572, Ep.Steps: 1215, Score: 278.68, Avg.Score: 188.61, Time: 07:41:06\n",
      "Ep.: 402, Total Steps: 470792, Ep.Steps: 1220, Score: 278.34, Avg.Score: 189.71, Time: 07:42:16\n",
      "Ep.: 403, Total Steps: 472080, Ep.Steps: 1288, Score: 273.38, Avg.Score: 190.31, Time: 07:43:31\n",
      "Ep.: 404, Total Steps: 473267, Ep.Steps: 1187, Score: 280.19, Avg.Score: 191.03, Time: 07:44:39\n",
      "Ep.: 405, Total Steps: 474509, Ep.Steps: 1242, Score: 277.23, Avg.Score: 191.68, Time: 07:45:51\n",
      "Ep.: 406, Total Steps: 475698, Ep.Steps: 1189, Score: 279.07, Avg.Score: 192.13, Time: 07:47:00\n",
      "Ep.: 407, Total Steps: 476955, Ep.Steps: 1257, Score: 274.35, Avg.Score: 192.78, Time: 07:48:13\n",
      "Ep.: 408, Total Steps: 478164, Ep.Steps: 1209, Score: 277.80, Avg.Score: 195.02, Time: 07:49:23\n",
      "Ep.: 409, Total Steps: 479430, Ep.Steps: 1266, Score: 277.09, Avg.Score: 195.75, Time: 07:50:36\n",
      "Ep.: 410, Total Steps: 480646, Ep.Steps: 1216, Score: 280.03, Avg.Score: 196.36, Time: 07:51:48\n",
      "Ep.: 411, Total Steps: 481825, Ep.Steps: 1179, Score: 281.62, Avg.Score: 199.50, Time: 07:52:56\n",
      "Ep.: 412, Total Steps: 482987, Ep.Steps: 1162, Score: 280.08, Avg.Score: 200.08, Time: 07:54:03\n",
      "Ep.: 413, Total Steps: 484193, Ep.Steps: 1206, Score: 275.94, Avg.Score: 200.64, Time: 07:55:13\n",
      "Ep.: 414, Total Steps: 484512, Ep.Steps: 319, Score: -47.02, Avg.Score: 199.30, Time: 07:55:32\n",
      "Ep.: 415, Total Steps: 485743, Ep.Steps: 1231, Score: 279.78, Avg.Score: 199.75, Time: 07:56:43\n",
      "Ep.: 416, Total Steps: 486945, Ep.Steps: 1202, Score: 279.00, Avg.Score: 200.09, Time: 07:57:53\n",
      "Ep.: 417, Total Steps: 488150, Ep.Steps: 1205, Score: 280.23, Avg.Score: 200.41, Time: 07:59:02\n",
      "Ep.: 418, Total Steps: 489272, Ep.Steps: 1122, Score: 285.20, Avg.Score: 203.63, Time: 08:00:08\n",
      "Ep.: 419, Total Steps: 490479, Ep.Steps: 1207, Score: 279.64, Avg.Score: 203.90, Time: 08:01:17\n",
      "Ep.: 420, Total Steps: 491710, Ep.Steps: 1231, Score: 281.14, Avg.Score: 204.18, Time: 08:02:29\n",
      "Ep.: 421, Total Steps: 492970, Ep.Steps: 1260, Score: 273.97, Avg.Score: 204.43, Time: 08:03:42\n",
      "Ep.: 422, Total Steps: 494177, Ep.Steps: 1207, Score: 280.22, Avg.Score: 204.71, Time: 08:04:52\n",
      "Ep.: 423, Total Steps: 495367, Ep.Steps: 1190, Score: 280.60, Avg.Score: 206.65, Time: 08:06:00\n",
      "Ep.: 424, Total Steps: 496524, Ep.Steps: 1157, Score: 283.86, Avg.Score: 206.89, Time: 08:07:07\n",
      "Ep.: 425, Total Steps: 497634, Ep.Steps: 1110, Score: 283.78, Avg.Score: 207.08, Time: 08:08:12\n",
      "Ep.: 426, Total Steps: 498919, Ep.Steps: 1285, Score: 273.34, Avg.Score: 207.21, Time: 08:09:26\n",
      "Ep.: 427, Total Steps: 500024, Ep.Steps: 1105, Score: 282.29, Avg.Score: 209.01, Time: 08:10:30\n",
      "Ep.: 428, Total Steps: 501128, Ep.Steps: 1104, Score: 282.63, Avg.Score: 212.63, Time: 08:11:34\n",
      "Ep.: 429, Total Steps: 502254, Ep.Steps: 1126, Score: 282.61, Avg.Score: 215.93, Time: 08:12:39\n",
      "Ep.: 430, Total Steps: 503373, Ep.Steps: 1119, Score: 284.91, Avg.Score: 219.63, Time: 08:13:44\n",
      "Ep.: 431, Total Steps: 504515, Ep.Steps: 1142, Score: 280.76, Avg.Score: 223.12, Time: 08:14:50\n",
      "Ep.: 432, Total Steps: 505628, Ep.Steps: 1113, Score: 284.14, Avg.Score: 226.11, Time: 08:15:55\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 433, Total Steps: 506737, Ep.Steps: 1109, Score: 284.92, Avg.Score: 229.95, Time: 08:16:59\n",
      "Ep.: 434, Total Steps: 507884, Ep.Steps: 1147, Score: 281.95, Avg.Score: 233.14, Time: 08:18:05\n",
      "Ep.: 435, Total Steps: 509013, Ep.Steps: 1129, Score: 281.95, Avg.Score: 236.58, Time: 08:19:10\n",
      "Ep.: 436, Total Steps: 510127, Ep.Steps: 1114, Score: 282.76, Avg.Score: 239.31, Time: 08:20:15\n",
      "Ep.: 437, Total Steps: 511248, Ep.Steps: 1121, Score: 283.72, Avg.Score: 242.39, Time: 08:21:20\n",
      "Ep.: 438, Total Steps: 512308, Ep.Steps: 1060, Score: 287.44, Avg.Score: 245.73, Time: 08:22:21\n",
      "Ep.: 439, Total Steps: 513357, Ep.Steps: 1049, Score: 286.72, Avg.Score: 247.42, Time: 08:23:22\n",
      "Ep.: 440, Total Steps: 514438, Ep.Steps: 1081, Score: 284.44, Avg.Score: 247.64, Time: 08:24:25\n",
      "Ep.: 441, Total Steps: 515510, Ep.Steps: 1072, Score: 285.61, Avg.Score: 247.89, Time: 08:25:27\n",
      "Ep.: 442, Total Steps: 516570, Ep.Steps: 1060, Score: 286.54, Avg.Score: 251.39, Time: 08:26:28\n",
      "Ep.: 443, Total Steps: 517621, Ep.Steps: 1051, Score: 286.02, Avg.Score: 251.69, Time: 08:27:29\n",
      "Ep.: 444, Total Steps: 518704, Ep.Steps: 1083, Score: 286.53, Avg.Score: 251.97, Time: 08:28:31\n",
      "Ep.: 445, Total Steps: 519790, Ep.Steps: 1086, Score: 285.87, Avg.Score: 252.36, Time: 08:29:34\n",
      "Ep.: 446, Total Steps: 520876, Ep.Steps: 1086, Score: 286.55, Avg.Score: 254.52, Time: 08:30:37\n",
      "Ep.: 447, Total Steps: 521956, Ep.Steps: 1080, Score: 283.94, Avg.Score: 257.74, Time: 08:31:40\n",
      "Ep.: 448, Total Steps: 523025, Ep.Steps: 1069, Score: 285.39, Avg.Score: 258.20, Time: 08:32:41\n",
      "Ep.: 449, Total Steps: 524077, Ep.Steps: 1052, Score: 286.05, Avg.Score: 262.25, Time: 08:33:42\n",
      "Ep.: 450, Total Steps: 525137, Ep.Steps: 1060, Score: 286.13, Avg.Score: 265.30, Time: 08:34:44\n",
      "Ep.: 451, Total Steps: 526236, Ep.Steps: 1099, Score: 283.66, Avg.Score: 265.87, Time: 08:35:47\n",
      "Ep.: 452, Total Steps: 527273, Ep.Steps: 1037, Score: 288.30, Avg.Score: 266.64, Time: 08:36:47\n",
      "Ep.: 453, Total Steps: 528309, Ep.Steps: 1036, Score: 287.72, Avg.Score: 268.34, Time: 08:37:48\n",
      "Ep.: 454, Total Steps: 529117, Ep.Steps: 808, Score: 131.77, Avg.Score: 267.12, Time: 08:38:35\n",
      "Ep.: 455, Total Steps: 530134, Ep.Steps: 1017, Score: 289.37, Avg.Score: 267.42, Time: 08:39:34\n",
      "Ep.: 456, Total Steps: 531174, Ep.Steps: 1040, Score: 286.95, Avg.Score: 267.74, Time: 08:40:34\n",
      "Ep.: 457, Total Steps: 532221, Ep.Steps: 1047, Score: 287.66, Avg.Score: 267.99, Time: 08:41:34\n",
      "Ep.: 458, Total Steps: 533279, Ep.Steps: 1058, Score: 287.32, Avg.Score: 268.35, Time: 08:42:36\n",
      "Ep.: 459, Total Steps: 534348, Ep.Steps: 1069, Score: 288.08, Avg.Score: 268.66, Time: 08:43:38\n",
      "Ep.: 460, Total Steps: 535383, Ep.Steps: 1035, Score: 288.05, Avg.Score: 270.38, Time: 08:44:37\n",
      "Ep.: 461, Total Steps: 536423, Ep.Steps: 1040, Score: 288.07, Avg.Score: 270.71, Time: 08:45:38\n",
      "Ep.: 462, Total Steps: 537450, Ep.Steps: 1027, Score: 290.02, Avg.Score: 271.05, Time: 08:46:37\n",
      "Ep.: 463, Total Steps: 538464, Ep.Steps: 1014, Score: 291.42, Avg.Score: 271.33, Time: 08:47:36\n",
      "Ep.: 464, Total Steps: 539540, Ep.Steps: 1076, Score: 287.56, Avg.Score: 271.54, Time: 08:48:38\n",
      "Ep.: 465, Total Steps: 540570, Ep.Steps: 1030, Score: 289.85, Avg.Score: 272.03, Time: 08:49:38\n",
      "Ep.: 466, Total Steps: 541588, Ep.Steps: 1018, Score: 290.27, Avg.Score: 272.29, Time: 08:50:36\n",
      "Ep.: 467, Total Steps: 542622, Ep.Steps: 1034, Score: 288.07, Avg.Score: 272.65, Time: 08:51:37\n",
      "Ep.: 468, Total Steps: 543631, Ep.Steps: 1009, Score: 291.01, Avg.Score: 274.30, Time: 08:52:36\n",
      "Ep.: 469, Total Steps: 544596, Ep.Steps: 965, Score: 293.76, Avg.Score: 274.63, Time: 08:53:32\n",
      "Ep.: 470, Total Steps: 545633, Ep.Steps: 1037, Score: 291.28, Avg.Score: 274.82, Time: 08:54:32\n",
      "Ep.: 471, Total Steps: 546683, Ep.Steps: 1050, Score: 287.31, Avg.Score: 275.03, Time: 08:55:33\n",
      "Ep.: 472, Total Steps: 547674, Ep.Steps: 991, Score: 292.23, Avg.Score: 275.30, Time: 08:56:31\n",
      "Ep.: 473, Total Steps: 548694, Ep.Steps: 1020, Score: 289.52, Avg.Score: 275.60, Time: 08:57:30\n",
      "Ep.: 474, Total Steps: 549715, Ep.Steps: 1021, Score: 290.86, Avg.Score: 275.85, Time: 08:58:30\n",
      "Ep.: 475, Total Steps: 550739, Ep.Steps: 1024, Score: 288.94, Avg.Score: 276.10, Time: 08:59:29\n",
      "Ep.: 476, Total Steps: 551813, Ep.Steps: 1074, Score: 287.57, Avg.Score: 276.53, Time: 09:00:32\n",
      "Ep.: 477, Total Steps: 552786, Ep.Steps: 973, Score: 293.32, Avg.Score: 276.81, Time: 09:01:28\n",
      "Ep.: 478, Total Steps: 553787, Ep.Steps: 1001, Score: 292.46, Avg.Score: 277.08, Time: 09:02:26\n",
      "Ep.: 479, Total Steps: 554807, Ep.Steps: 1020, Score: 290.29, Avg.Score: 277.25, Time: 09:03:25\n",
      "Ep.: 480, Total Steps: 555790, Ep.Steps: 983, Score: 293.31, Avg.Score: 277.46, Time: 09:04:22\n",
      "Ep.: 481, Total Steps: 556783, Ep.Steps: 993, Score: 293.37, Avg.Score: 277.63, Time: 09:05:20\n",
      "Ep.: 482, Total Steps: 557763, Ep.Steps: 980, Score: 292.56, Avg.Score: 277.84, Time: 09:06:16\n",
      "Ep.: 483, Total Steps: 558734, Ep.Steps: 971, Score: 294.85, Avg.Score: 278.05, Time: 09:07:13\n",
      "Ep.: 484, Total Steps: 559733, Ep.Steps: 999, Score: 291.76, Avg.Score: 278.22, Time: 09:08:11\n",
      "Ep.: 485, Total Steps: 560745, Ep.Steps: 1012, Score: 290.06, Avg.Score: 278.35, Time: 09:09:09\n",
      "Ep.: 486, Total Steps: 561671, Ep.Steps: 926, Score: 295.62, Avg.Score: 278.56, Time: 09:10:03\n",
      "Ep.: 487, Total Steps: 562602, Ep.Steps: 931, Score: 295.24, Avg.Score: 278.80, Time: 09:10:57\n",
      "Ep.: 488, Total Steps: 563606, Ep.Steps: 1004, Score: 292.90, Avg.Score: 279.01, Time: 09:11:55\n",
      "Ep.: 489, Total Steps: 564568, Ep.Steps: 962, Score: 293.77, Avg.Score: 279.20, Time: 09:12:51\n",
      "Ep.: 490, Total Steps: 565523, Ep.Steps: 955, Score: 294.77, Avg.Score: 279.39, Time: 09:13:47\n",
      "Ep.: 491, Total Steps: 566475, Ep.Steps: 952, Score: 293.79, Avg.Score: 279.70, Time: 09:14:42\n",
      "Ep.: 492, Total Steps: 567405, Ep.Steps: 930, Score: 294.54, Avg.Score: 279.91, Time: 09:15:36\n",
      "Ep.: 493, Total Steps: 568356, Ep.Steps: 951, Score: 294.46, Avg.Score: 280.21, Time: 09:16:31\n",
      "Ep.: 494, Total Steps: 569334, Ep.Steps: 978, Score: 294.48, Avg.Score: 280.46, Time: 09:17:28\n",
      "Ep.: 495, Total Steps: 570306, Ep.Steps: 972, Score: 292.73, Avg.Score: 280.69, Time: 09:18:24\n",
      "Ep.: 496, Total Steps: 571267, Ep.Steps: 961, Score: 291.61, Avg.Score: 280.85, Time: 09:19:19\n",
      "Ep.: 497, Total Steps: 572247, Ep.Steps: 980, Score: 291.20, Avg.Score: 281.06, Time: 09:20:16\n",
      "Ep.: 498, Total Steps: 573176, Ep.Steps: 929, Score: 293.41, Avg.Score: 281.20, Time: 09:21:10\n",
      "Ep.: 499, Total Steps: 574099, Ep.Steps: 923, Score: 295.22, Avg.Score: 281.41, Time: 09:22:04\n",
      "Ep.: 500, Total Steps: 575046, Ep.Steps: 947, Score: 292.32, Avg.Score: 281.52, Time: 09:22:59\n",
      "Ep.: 501, Total Steps: 575975, Ep.Steps: 929, Score: 294.34, Avg.Score: 281.68, Time: 09:23:53\n",
      "Ep.: 502, Total Steps: 576934, Ep.Steps: 959, Score: 292.50, Avg.Score: 281.82, Time: 09:24:48\n",
      "Ep.: 503, Total Steps: 577865, Ep.Steps: 931, Score: 294.54, Avg.Score: 282.03, Time: 09:25:42\n",
      "Ep.: 504, Total Steps: 578791, Ep.Steps: 926, Score: 295.09, Avg.Score: 282.18, Time: 09:26:36\n",
      "Ep.: 505, Total Steps: 579752, Ep.Steps: 961, Score: 293.87, Avg.Score: 282.35, Time: 09:27:31\n",
      "Ep.: 506, Total Steps: 580703, Ep.Steps: 951, Score: 293.73, Avg.Score: 282.49, Time: 09:28:26\n",
      "Ep.: 507, Total Steps: 581651, Ep.Steps: 948, Score: 295.39, Avg.Score: 282.70, Time: 09:29:21\n",
      "Ep.: 508, Total Steps: 582633, Ep.Steps: 982, Score: 294.02, Avg.Score: 282.87, Time: 09:30:18\n",
      "Ep.: 509, Total Steps: 583578, Ep.Steps: 945, Score: 296.71, Avg.Score: 283.06, Time: 09:31:13\n",
      "Ep.: 510, Total Steps: 584506, Ep.Steps: 928, Score: 295.74, Avg.Score: 283.22, Time: 09:32:07\n",
      "Ep.: 511, Total Steps: 585432, Ep.Steps: 926, Score: 296.68, Avg.Score: 283.37, Time: 09:33:00\n",
      "Ep.: 512, Total Steps: 586376, Ep.Steps: 944, Score: 296.34, Avg.Score: 283.53, Time: 09:33:55\n",
      "Ep.: 513, Total Steps: 587289, Ep.Steps: 913, Score: 294.17, Avg.Score: 283.71, Time: 09:34:48\n",
      "Ep.: 514, Total Steps: 588209, Ep.Steps: 920, Score: 294.68, Avg.Score: 287.13, Time: 09:35:41\n",
      "Ep.: 515, Total Steps: 589148, Ep.Steps: 939, Score: 292.79, Avg.Score: 287.26, Time: 09:36:36\n",
      "Ep.: 516, Total Steps: 590130, Ep.Steps: 982, Score: 293.89, Avg.Score: 287.41, Time: 09:37:33\n",
      "Ep.: 517, Total Steps: 591032, Ep.Steps: 902, Score: 296.28, Avg.Score: 287.57, Time: 09:38:25\n",
      "Ep.: 518, Total Steps: 591954, Ep.Steps: 922, Score: 296.10, Avg.Score: 287.68, Time: 09:39:18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 519, Total Steps: 592877, Ep.Steps: 923, Score: 297.95, Avg.Score: 287.86, Time: 09:40:12\n",
      "Ep.: 520, Total Steps: 593407, Ep.Steps: 530, Score: 60.62, Avg.Score: 285.66, Time: 09:40:43\n",
      "Ep.: 521, Total Steps: 594295, Ep.Steps: 888, Score: 298.59, Avg.Score: 285.90, Time: 09:41:34\n",
      "Ep.: 522, Total Steps: 595202, Ep.Steps: 907, Score: 296.86, Avg.Score: 286.07, Time: 09:42:27\n",
      "Ep.: 523, Total Steps: 596115, Ep.Steps: 913, Score: 297.53, Avg.Score: 286.24, Time: 09:43:20\n",
      "Ep.: 524, Total Steps: 597038, Ep.Steps: 923, Score: 295.96, Avg.Score: 286.36, Time: 09:44:13\n",
      "Ep.: 525, Total Steps: 597933, Ep.Steps: 895, Score: 293.82, Avg.Score: 286.46, Time: 09:45:05\n",
      "Ep.: 526, Total Steps: 598828, Ep.Steps: 895, Score: 294.79, Avg.Score: 286.68, Time: 09:45:57\n",
      "Ep.: 527, Total Steps: 599762, Ep.Steps: 934, Score: 295.75, Avg.Score: 286.81, Time: 09:46:51\n",
      "Ep.: 528, Total Steps: 600678, Ep.Steps: 916, Score: 294.86, Avg.Score: 286.93, Time: 09:47:44\n",
      "Ep.: 529, Total Steps: 601612, Ep.Steps: 934, Score: 293.44, Avg.Score: 287.04, Time: 09:48:38\n",
      "Ep.: 530, Total Steps: 602509, Ep.Steps: 897, Score: 296.61, Avg.Score: 287.16, Time: 09:49:30\n",
      "Ep.: 531, Total Steps: 603383, Ep.Steps: 874, Score: 297.55, Avg.Score: 287.33, Time: 09:50:20\n",
      "Ep.: 532, Total Steps: 604289, Ep.Steps: 906, Score: 295.75, Avg.Score: 287.44, Time: 09:51:13\n",
      "Ep.: 533, Total Steps: 605184, Ep.Steps: 895, Score: 295.73, Avg.Score: 287.55, Time: 09:52:05\n",
      "Ep.: 534, Total Steps: 606104, Ep.Steps: 920, Score: 296.25, Avg.Score: 287.69, Time: 09:52:58\n",
      "Ep.: 535, Total Steps: 607021, Ep.Steps: 917, Score: 295.77, Avg.Score: 287.83, Time: 09:53:52\n",
      "Ep.: 536, Total Steps: 607930, Ep.Steps: 909, Score: 295.68, Avg.Score: 287.96, Time: 09:54:44\n",
      "Ep.: 537, Total Steps: 608846, Ep.Steps: 916, Score: 296.66, Avg.Score: 288.09, Time: 09:55:37\n",
      "Ep.: 538, Total Steps: 609748, Ep.Steps: 902, Score: 296.43, Avg.Score: 288.18, Time: 09:56:30\n",
      "Ep.: 539, Total Steps: 610660, Ep.Steps: 912, Score: 296.41, Avg.Score: 288.28, Time: 09:57:23\n",
      "Ep.: 540, Total Steps: 611574, Ep.Steps: 914, Score: 297.10, Avg.Score: 288.40, Time: 09:58:16\n",
      "Ep.: 541, Total Steps: 612491, Ep.Steps: 917, Score: 296.22, Avg.Score: 288.51, Time: 09:59:09\n",
      "Ep.: 542, Total Steps: 613393, Ep.Steps: 902, Score: 296.12, Avg.Score: 288.60, Time: 10:00:02\n",
      "Ep.: 543, Total Steps: 614278, Ep.Steps: 885, Score: 296.06, Avg.Score: 288.71, Time: 10:00:54\n",
      "Ep.: 544, Total Steps: 615198, Ep.Steps: 920, Score: 295.66, Avg.Score: 288.80, Time: 10:01:47\n",
      "Ep.: 545, Total Steps: 616080, Ep.Steps: 882, Score: 297.75, Avg.Score: 288.92, Time: 10:02:38\n",
      "Ep.: 546, Total Steps: 616992, Ep.Steps: 912, Score: 297.25, Avg.Score: 289.02, Time: 10:03:31\n",
      "Ep.: 547, Total Steps: 617917, Ep.Steps: 925, Score: 296.59, Avg.Score: 289.15, Time: 10:04:25\n",
      "Ep.: 548, Total Steps: 618819, Ep.Steps: 902, Score: 296.79, Avg.Score: 289.26, Time: 10:05:17\n",
      "Ep.: 549, Total Steps: 619719, Ep.Steps: 900, Score: 296.91, Avg.Score: 289.37, Time: 10:06:09\n",
      "Ep.: 550, Total Steps: 620626, Ep.Steps: 907, Score: 299.16, Avg.Score: 289.50, Time: 10:07:01\n",
      "Ep.: 551, Total Steps: 621539, Ep.Steps: 913, Score: 296.49, Avg.Score: 289.63, Time: 10:07:55\n",
      "Ep.: 552, Total Steps: 622431, Ep.Steps: 892, Score: 299.07, Avg.Score: 289.74, Time: 10:08:46\n",
      "Ep.: 553, Total Steps: 623322, Ep.Steps: 891, Score: 297.19, Avg.Score: 289.83, Time: 10:09:38\n",
      "Ep.: 554, Total Steps: 624226, Ep.Steps: 904, Score: 298.25, Avg.Score: 291.50, Time: 10:10:31\n",
      "Ep.: 555, Total Steps: 625114, Ep.Steps: 888, Score: 297.96, Avg.Score: 291.58, Time: 10:11:23\n",
      "Ep.: 556, Total Steps: 625993, Ep.Steps: 879, Score: 297.43, Avg.Score: 291.69, Time: 10:12:13\n",
      "Ep.: 557, Total Steps: 626894, Ep.Steps: 901, Score: 295.58, Avg.Score: 291.77, Time: 10:13:06\n",
      "Ep.: 558, Total Steps: 627806, Ep.Steps: 912, Score: 297.50, Avg.Score: 291.87, Time: 10:13:58\n",
      "Ep.: 559, Total Steps: 628699, Ep.Steps: 893, Score: 295.66, Avg.Score: 291.94, Time: 10:14:50\n",
      "Ep.: 560, Total Steps: 629579, Ep.Steps: 880, Score: 296.76, Avg.Score: 292.03, Time: 10:15:41\n",
      "Ep.: 561, Total Steps: 630469, Ep.Steps: 890, Score: 297.07, Avg.Score: 292.12, Time: 10:16:34\n",
      "Ep.: 562, Total Steps: 631356, Ep.Steps: 887, Score: 295.37, Avg.Score: 292.18, Time: 10:17:27\n",
      "Ep.: 563, Total Steps: 632225, Ep.Steps: 869, Score: 298.48, Avg.Score: 292.25, Time: 10:18:19\n",
      "Ep.: 564, Total Steps: 633135, Ep.Steps: 910, Score: 296.74, Avg.Score: 292.34, Time: 10:19:16\n",
      "Ep.: 565, Total Steps: 634034, Ep.Steps: 899, Score: 296.63, Avg.Score: 292.41, Time: 10:20:09\n",
      "Ep.: 566, Total Steps: 634933, Ep.Steps: 899, Score: 298.12, Avg.Score: 292.48, Time: 10:21:06\n",
      "Ep.: 567, Total Steps: 635837, Ep.Steps: 904, Score: 296.77, Avg.Score: 292.57, Time: 10:22:00\n",
      "Ep.: 568, Total Steps: 636722, Ep.Steps: 885, Score: 296.85, Avg.Score: 292.63, Time: 10:22:54\n",
      "Ep.: 569, Total Steps: 637606, Ep.Steps: 884, Score: 296.58, Avg.Score: 292.66, Time: 10:23:48\n",
      "Ep.: 570, Total Steps: 638511, Ep.Steps: 905, Score: 297.71, Avg.Score: 292.72, Time: 10:24:47\n",
      "Ep.: 571, Total Steps: 639389, Ep.Steps: 878, Score: 295.63, Avg.Score: 292.80, Time: 10:25:42\n",
      "Ep.: 572, Total Steps: 640256, Ep.Steps: 867, Score: 296.74, Avg.Score: 292.85, Time: 10:26:43\n",
      "Ep.: 573, Total Steps: 641156, Ep.Steps: 900, Score: 295.83, Avg.Score: 292.91, Time: 10:27:39\n",
      "Ep.: 574, Total Steps: 642101, Ep.Steps: 945, Score: 296.43, Avg.Score: 292.97, Time: 10:28:49\n",
      "Ep.: 575, Total Steps: 642987, Ep.Steps: 886, Score: 296.89, Avg.Score: 293.05, Time: 10:29:52\n",
      "Ep.: 576, Total Steps: 643919, Ep.Steps: 932, Score: 296.11, Avg.Score: 293.13, Time: 10:30:56\n",
      "Ep.: 577, Total Steps: 644814, Ep.Steps: 895, Score: 297.23, Avg.Score: 293.17, Time: 10:31:50\n",
      "Ep.: 578, Total Steps: 645755, Ep.Steps: 941, Score: 295.24, Avg.Score: 293.20, Time: 10:32:48\n",
      "Ep.: 579, Total Steps: 646625, Ep.Steps: 870, Score: 297.75, Avg.Score: 293.28, Time: 10:33:40\n",
      "Ep.: 580, Total Steps: 647527, Ep.Steps: 902, Score: 296.66, Avg.Score: 293.31, Time: 10:34:35\n",
      "Ep.: 581, Total Steps: 648416, Ep.Steps: 889, Score: 296.10, Avg.Score: 293.34, Time: 10:35:31\n",
      "Ep.: 582, Total Steps: 649287, Ep.Steps: 871, Score: 295.80, Avg.Score: 293.37, Time: 10:36:24\n",
      "Ep.: 583, Total Steps: 650194, Ep.Steps: 907, Score: 296.83, Avg.Score: 293.39, Time: 10:37:17\n",
      "Ep.: 584, Total Steps: 651074, Ep.Steps: 880, Score: 296.50, Avg.Score: 293.44, Time: 10:38:09\n",
      "Ep.: 585, Total Steps: 651979, Ep.Steps: 905, Score: 296.89, Avg.Score: 293.50, Time: 10:39:02\n",
      "Ep.: 586, Total Steps: 652836, Ep.Steps: 857, Score: 300.11, Avg.Score: 293.55, Time: 10:39:52\n",
      "Ep.: 587, Total Steps: 653728, Ep.Steps: 892, Score: 297.02, Avg.Score: 293.57, Time: 10:40:44\n",
      "Ep.: 588, Total Steps: 654613, Ep.Steps: 885, Score: 297.52, Avg.Score: 293.61, Time: 10:41:36\n",
      "Ep.: 589, Total Steps: 655536, Ep.Steps: 923, Score: 295.85, Avg.Score: 293.63, Time: 10:42:29\n",
      "Ep.: 590, Total Steps: 656473, Ep.Steps: 937, Score: 295.92, Avg.Score: 293.65, Time: 10:43:24\n",
      "Ep.: 591, Total Steps: 657358, Ep.Steps: 885, Score: 296.96, Avg.Score: 293.68, Time: 10:44:18\n",
      "Ep.: 592, Total Steps: 658262, Ep.Steps: 904, Score: 297.80, Avg.Score: 293.71, Time: 10:45:11\n",
      "Ep.: 593, Total Steps: 659132, Ep.Steps: 870, Score: 298.29, Avg.Score: 293.75, Time: 10:46:02\n",
      "Ep.: 594, Total Steps: 660018, Ep.Steps: 886, Score: 298.22, Avg.Score: 293.79, Time: 10:46:54\n",
      "Ep.: 595, Total Steps: 660898, Ep.Steps: 880, Score: 297.95, Avg.Score: 293.84, Time: 10:47:45\n",
      "Ep.: 596, Total Steps: 661781, Ep.Steps: 883, Score: 297.56, Avg.Score: 293.90, Time: 10:48:37\n",
      "Ep.: 597, Total Steps: 662684, Ep.Steps: 903, Score: 296.89, Avg.Score: 293.95, Time: 10:49:30\n",
      "Ep.: 598, Total Steps: 663571, Ep.Steps: 887, Score: 299.91, Avg.Score: 294.02, Time: 10:50:22\n",
      "Ep.: 599, Total Steps: 664446, Ep.Steps: 875, Score: 297.12, Avg.Score: 294.04, Time: 10:51:13\n",
      "Ep.: 600, Total Steps: 665322, Ep.Steps: 876, Score: 298.92, Avg.Score: 294.10, Time: 10:52:04\n",
      "Ep.: 601, Total Steps: 666196, Ep.Steps: 874, Score: 298.11, Avg.Score: 294.14, Time: 10:52:56\n",
      "Ep.: 602, Total Steps: 667085, Ep.Steps: 889, Score: 298.34, Avg.Score: 294.20, Time: 10:53:48\n",
      "Ep.: 603, Total Steps: 667974, Ep.Steps: 889, Score: 296.53, Avg.Score: 294.22, Time: 10:54:40\n",
      "Ep.: 604, Total Steps: 668885, Ep.Steps: 911, Score: 297.01, Avg.Score: 294.24, Time: 10:55:33\n",
      "Ep.: 605, Total Steps: 669785, Ep.Steps: 900, Score: 297.18, Avg.Score: 294.27, Time: 10:56:26\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 606, Total Steps: 670686, Ep.Steps: 901, Score: 298.26, Avg.Score: 294.32, Time: 10:57:18\n",
      "Ep.: 607, Total Steps: 671566, Ep.Steps: 880, Score: 296.87, Avg.Score: 294.33, Time: 10:58:10\n",
      "Ep.: 608, Total Steps: 672451, Ep.Steps: 885, Score: 299.22, Avg.Score: 294.38, Time: 10:59:03\n",
      "Ep.: 609, Total Steps: 673316, Ep.Steps: 865, Score: 298.95, Avg.Score: 294.41, Time: 10:59:53\n",
      "Ep.: 610, Total Steps: 674169, Ep.Steps: 853, Score: 297.32, Avg.Score: 294.42, Time: 11:00:46\n",
      "Ep.: 611, Total Steps: 675028, Ep.Steps: 859, Score: 297.11, Avg.Score: 294.43, Time: 11:01:37\n",
      "Ep.: 612, Total Steps: 675870, Ep.Steps: 842, Score: 298.94, Avg.Score: 294.45, Time: 11:02:27\n",
      "Ep.: 613, Total Steps: 676187, Ep.Steps: 317, Score: -18.50, Avg.Score: 291.33, Time: 11:02:46\n",
      "Ep.: 614, Total Steps: 677033, Ep.Steps: 846, Score: 296.49, Avg.Score: 291.34, Time: 11:03:35\n",
      "Ep.: 615, Total Steps: 677909, Ep.Steps: 876, Score: 297.99, Avg.Score: 291.40, Time: 11:04:26\n",
      "Ep.: 616, Total Steps: 678797, Ep.Steps: 888, Score: 298.00, Avg.Score: 291.44, Time: 11:05:18\n",
      "Ep.: 617, Total Steps: 679668, Ep.Steps: 871, Score: 297.91, Avg.Score: 291.45, Time: 11:06:08\n",
      "Ep.: 618, Total Steps: 680524, Ep.Steps: 856, Score: 296.78, Avg.Score: 291.46, Time: 11:06:58\n",
      "Ep.: 619, Total Steps: 681399, Ep.Steps: 875, Score: 295.90, Avg.Score: 291.44, Time: 11:07:49\n",
      "Ep.: 620, Total Steps: 682264, Ep.Steps: 865, Score: 297.21, Avg.Score: 293.81, Time: 11:08:40\n",
      "Ep.: 621, Total Steps: 683159, Ep.Steps: 895, Score: 298.46, Avg.Score: 293.80, Time: 11:09:32\n",
      "Ep.: 622, Total Steps: 684003, Ep.Steps: 844, Score: 299.07, Avg.Score: 293.83, Time: 11:10:23\n",
      "Ep.: 623, Total Steps: 684847, Ep.Steps: 844, Score: 298.73, Avg.Score: 293.84, Time: 11:11:19\n",
      "Ep.: 624, Total Steps: 685686, Ep.Steps: 839, Score: 299.25, Avg.Score: 293.87, Time: 11:12:11\n",
      "Ep.: 625, Total Steps: 686569, Ep.Steps: 883, Score: 299.21, Avg.Score: 293.92, Time: 11:13:07\n",
      "Ep.: 626, Total Steps: 687443, Ep.Steps: 874, Score: 298.98, Avg.Score: 293.97, Time: 11:14:02\n",
      "Ep.: 627, Total Steps: 688315, Ep.Steps: 872, Score: 301.03, Avg.Score: 294.02, Time: 11:14:55\n",
      "Ep.: 628, Total Steps: 689165, Ep.Steps: 850, Score: 301.23, Avg.Score: 294.08, Time: 11:15:48\n",
      "Ep.: 629, Total Steps: 690004, Ep.Steps: 839, Score: 301.16, Avg.Score: 294.16, Time: 11:16:40\n",
      "Ep.: 630, Total Steps: 690889, Ep.Steps: 885, Score: 298.33, Avg.Score: 294.18, Time: 11:17:34\n",
      "Ep.: 631, Total Steps: 691752, Ep.Steps: 863, Score: 297.96, Avg.Score: 294.18, Time: 11:18:28\n",
      "Ep.: 632, Total Steps: 692605, Ep.Steps: 853, Score: 298.45, Avg.Score: 294.21, Time: 11:19:20\n",
      "Ep.: 633, Total Steps: 693464, Ep.Steps: 859, Score: 300.29, Avg.Score: 294.25, Time: 11:20:13\n",
      "Ep.: 634, Total Steps: 694346, Ep.Steps: 882, Score: 295.58, Avg.Score: 294.25, Time: 11:21:08\n",
      "Ep.: 635, Total Steps: 695190, Ep.Steps: 844, Score: 300.08, Avg.Score: 294.29, Time: 11:22:00\n",
      "Ep.: 636, Total Steps: 696055, Ep.Steps: 865, Score: 298.08, Avg.Score: 294.31, Time: 11:22:53\n",
      "Ep.: 637, Total Steps: 696935, Ep.Steps: 880, Score: 298.21, Avg.Score: 294.33, Time: 11:23:48\n",
      "Ep.: 638, Total Steps: 697807, Ep.Steps: 872, Score: 299.99, Avg.Score: 294.37, Time: 11:24:41\n",
      "Ep.: 639, Total Steps: 698656, Ep.Steps: 849, Score: 300.57, Avg.Score: 294.41, Time: 11:25:34\n",
      "Ep.: 640, Total Steps: 699527, Ep.Steps: 871, Score: 298.85, Avg.Score: 294.43, Time: 11:26:27\n",
      "Ep.: 641, Total Steps: 700390, Ep.Steps: 863, Score: 299.36, Avg.Score: 294.46, Time: 11:27:20\n",
      "Ep.: 642, Total Steps: 701237, Ep.Steps: 847, Score: 300.80, Avg.Score: 294.50, Time: 11:28:12\n",
      "Ep.: 643, Total Steps: 702076, Ep.Steps: 839, Score: 300.42, Avg.Score: 294.55, Time: 11:29:04\n",
      "Ep.: 644, Total Steps: 702946, Ep.Steps: 870, Score: 300.67, Avg.Score: 294.60, Time: 11:29:57\n",
      "Ep.: 645, Total Steps: 703818, Ep.Steps: 872, Score: 301.34, Avg.Score: 294.63, Time: 11:30:51\n",
      "Ep.: 646, Total Steps: 704671, Ep.Steps: 853, Score: 303.17, Avg.Score: 294.69, Time: 11:31:43\n",
      "Ep.: 647, Total Steps: 705527, Ep.Steps: 856, Score: 301.25, Avg.Score: 294.74, Time: 11:32:35\n",
      "Ep.: 648, Total Steps: 706375, Ep.Steps: 848, Score: 301.10, Avg.Score: 294.78, Time: 11:33:28\n",
      "Ep.: 649, Total Steps: 707272, Ep.Steps: 897, Score: 299.08, Avg.Score: 294.80, Time: 11:34:23\n",
      "Ep.: 650, Total Steps: 708127, Ep.Steps: 855, Score: 301.40, Avg.Score: 294.83, Time: 11:35:15\n",
      "Ep.: 651, Total Steps: 708994, Ep.Steps: 867, Score: 301.52, Avg.Score: 294.88, Time: 11:36:08\n",
      "Ep.: 652, Total Steps: 709834, Ep.Steps: 840, Score: 301.07, Avg.Score: 294.90, Time: 11:37:00\n",
      "Ep.: 653, Total Steps: 710668, Ep.Steps: 834, Score: 304.62, Avg.Score: 294.97, Time: 11:37:51\n",
      "Ep.: 654, Total Steps: 711515, Ep.Steps: 847, Score: 301.51, Avg.Score: 295.00, Time: 11:38:43\n",
      "Ep.: 655, Total Steps: 712361, Ep.Steps: 846, Score: 301.29, Avg.Score: 295.04, Time: 11:39:35\n",
      "Ep.: 656, Total Steps: 713214, Ep.Steps: 853, Score: 301.40, Avg.Score: 295.08, Time: 11:40:28\n",
      "Ep.: 657, Total Steps: 714084, Ep.Steps: 870, Score: 301.50, Avg.Score: 295.14, Time: 11:41:21\n",
      "Ep.: 658, Total Steps: 714953, Ep.Steps: 869, Score: 302.21, Avg.Score: 295.18, Time: 11:42:14\n",
      "Ep.: 659, Total Steps: 715813, Ep.Steps: 860, Score: 302.17, Avg.Score: 295.25, Time: 11:43:07\n",
      "Ep.: 660, Total Steps: 716650, Ep.Steps: 837, Score: 303.10, Avg.Score: 295.31, Time: 11:43:58\n",
      "Ep.: 661, Total Steps: 717508, Ep.Steps: 858, Score: 302.46, Avg.Score: 295.36, Time: 11:44:51\n",
      "Ep.: 662, Total Steps: 718368, Ep.Steps: 860, Score: 302.38, Avg.Score: 295.43, Time: 11:45:44\n",
      "Ep.: 663, Total Steps: 719218, Ep.Steps: 850, Score: 302.98, Avg.Score: 295.48, Time: 11:46:36\n",
      "Ep.: 664, Total Steps: 720074, Ep.Steps: 856, Score: 301.28, Avg.Score: 295.53, Time: 11:47:29\n",
      "Ep.: 665, Total Steps: 720911, Ep.Steps: 837, Score: 301.73, Avg.Score: 295.58, Time: 11:48:20\n",
      "Ep.: 666, Total Steps: 721314, Ep.Steps: 403, Score: 32.08, Avg.Score: 292.92, Time: 11:48:44\n",
      "Ep.: 667, Total Steps: 722190, Ep.Steps: 876, Score: 301.82, Avg.Score: 292.97, Time: 11:49:38\n",
      "Ep.: 668, Total Steps: 723047, Ep.Steps: 857, Score: 301.90, Avg.Score: 293.02, Time: 11:50:31\n",
      "Ep.: 669, Total Steps: 723874, Ep.Steps: 827, Score: 303.29, Avg.Score: 293.08, Time: 11:51:21\n",
      "Ep.: 670, Total Steps: 724738, Ep.Steps: 864, Score: 302.27, Avg.Score: 293.13, Time: 11:52:14\n",
      "Ep.: 671, Total Steps: 725623, Ep.Steps: 885, Score: 300.24, Avg.Score: 293.18, Time: 11:53:09\n",
      "Ep.: 672, Total Steps: 726474, Ep.Steps: 851, Score: 300.75, Avg.Score: 293.22, Time: 11:54:01\n",
      "Ep.: 673, Total Steps: 727308, Ep.Steps: 834, Score: 302.73, Avg.Score: 293.28, Time: 11:54:53\n",
      "Ep.: 674, Total Steps: 728146, Ep.Steps: 838, Score: 302.07, Avg.Score: 293.34, Time: 11:55:44\n",
      "Ep.: 675, Total Steps: 729000, Ep.Steps: 854, Score: 302.11, Avg.Score: 293.39, Time: 11:56:37\n",
      "Ep.: 676, Total Steps: 729858, Ep.Steps: 858, Score: 302.64, Avg.Score: 293.46, Time: 11:57:29\n",
      "Ep.: 677, Total Steps: 730716, Ep.Steps: 858, Score: 303.64, Avg.Score: 293.52, Time: 11:58:22\n",
      "Ep.: 678, Total Steps: 731592, Ep.Steps: 876, Score: 302.46, Avg.Score: 293.60, Time: 11:59:17\n",
      "Ep.: 679, Total Steps: 732480, Ep.Steps: 888, Score: 301.26, Avg.Score: 293.63, Time: 12:00:12\n",
      "Ep.: 680, Total Steps: 733348, Ep.Steps: 868, Score: 299.79, Avg.Score: 293.66, Time: 12:01:05\n",
      "Ep.: 681, Total Steps: 734230, Ep.Steps: 882, Score: 301.06, Avg.Score: 293.71, Time: 12:01:59\n",
      "Ep.: 682, Total Steps: 735111, Ep.Steps: 881, Score: 301.63, Avg.Score: 293.77, Time: 12:02:53\n",
      "Ep.: 683, Total Steps: 735975, Ep.Steps: 864, Score: 301.53, Avg.Score: 293.82, Time: 12:03:46\n",
      "Ep.: 684, Total Steps: 736821, Ep.Steps: 846, Score: 305.24, Avg.Score: 293.90, Time: 12:04:38\n",
      "Ep.: 685, Total Steps: 737679, Ep.Steps: 858, Score: 302.24, Avg.Score: 293.96, Time: 12:05:31\n",
      "Ep.: 686, Total Steps: 738547, Ep.Steps: 868, Score: 300.96, Avg.Score: 293.97, Time: 12:06:25\n",
      "Ep.: 687, Total Steps: 739399, Ep.Steps: 852, Score: 301.49, Avg.Score: 294.01, Time: 12:07:17\n",
      "Ep.: 688, Total Steps: 740252, Ep.Steps: 853, Score: 300.39, Avg.Score: 294.04, Time: 12:08:09\n",
      "Ep.: 689, Total Steps: 741113, Ep.Steps: 861, Score: 302.82, Avg.Score: 294.11, Time: 12:09:02\n",
      "Ep.: 690, Total Steps: 741948, Ep.Steps: 835, Score: 302.66, Avg.Score: 294.18, Time: 12:09:53\n",
      "Ep.: 691, Total Steps: 742805, Ep.Steps: 857, Score: 302.03, Avg.Score: 294.23, Time: 12:10:46\n",
      "Ep.: 692, Total Steps: 743660, Ep.Steps: 855, Score: 302.83, Avg.Score: 294.28, Time: 12:11:38\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ep.: 693, Total Steps: 744548, Ep.Steps: 888, Score: 302.08, Avg.Score: 294.32, Time: 12:12:31\n",
      "Ep.: 694, Total Steps: 745409, Ep.Steps: 861, Score: 303.74, Avg.Score: 294.37, Time: 12:13:23\n",
      "Ep.: 695, Total Steps: 746255, Ep.Steps: 846, Score: 304.20, Avg.Score: 294.43, Time: 12:14:13\n",
      "Ep.: 696, Total Steps: 747095, Ep.Steps: 840, Score: 305.14, Avg.Score: 294.51, Time: 12:15:04\n",
      "Ep.: 697, Total Steps: 747903, Ep.Steps: 808, Score: 305.75, Avg.Score: 294.60, Time: 12:15:52\n",
      "Ep.: 698, Total Steps: 748765, Ep.Steps: 862, Score: 301.62, Avg.Score: 294.61, Time: 12:16:44\n",
      "Ep.: 699, Total Steps: 749626, Ep.Steps: 861, Score: 302.43, Avg.Score: 294.67, Time: 12:17:36\n",
      "Ep.: 700, Total Steps: 750487, Ep.Steps: 861, Score: 303.67, Avg.Score: 294.72, Time: 12:18:27\n",
      "Ep.: 701, Total Steps: 751349, Ep.Steps: 862, Score: 304.16, Avg.Score: 294.78, Time: 12:19:19\n",
      "Ep.: 702, Total Steps: 752207, Ep.Steps: 858, Score: 302.79, Avg.Score: 294.82, Time: 12:20:11\n",
      "Ep.: 703, Total Steps: 753044, Ep.Steps: 837, Score: 301.42, Avg.Score: 294.87, Time: 12:21:01\n",
      "Ep.: 704, Total Steps: 753921, Ep.Steps: 877, Score: 303.58, Avg.Score: 294.93, Time: 12:21:54\n",
      "Ep.: 705, Total Steps: 754773, Ep.Steps: 852, Score: 303.06, Avg.Score: 294.99, Time: 12:22:45\n",
      "Ep.: 706, Total Steps: 755648, Ep.Steps: 875, Score: 300.55, Avg.Score: 295.02, Time: 12:23:38\n",
      "Ep.: 707, Total Steps: 756505, Ep.Steps: 857, Score: 302.50, Avg.Score: 295.07, Time: 12:24:29\n",
      "Ep.: 708, Total Steps: 757384, Ep.Steps: 879, Score: 302.39, Avg.Score: 295.10, Time: 12:25:22\n",
      "Ep.: 709, Total Steps: 758213, Ep.Steps: 829, Score: 301.78, Avg.Score: 295.13, Time: 12:26:12\n",
      "Ep.: 710, Total Steps: 759068, Ep.Steps: 855, Score: 303.71, Avg.Score: 295.20, Time: 12:27:03\n",
      "Ep.: 711, Total Steps: 759927, Ep.Steps: 859, Score: 302.86, Avg.Score: 295.25, Time: 12:27:55\n",
      "Ep.: 712, Total Steps: 760769, Ep.Steps: 842, Score: 304.20, Avg.Score: 295.31, Time: 12:28:45\n",
      "Ep.: 713, Total Steps: 761626, Ep.Steps: 857, Score: 303.63, Avg.Score: 298.53, Time: 12:29:37\n",
      "Ep.: 714, Total Steps: 762462, Ep.Steps: 836, Score: 305.25, Avg.Score: 298.62, Time: 12:30:27\n",
      "Ep.: 715, Total Steps: 763310, Ep.Steps: 848, Score: 305.02, Avg.Score: 298.69, Time: 12:31:18\n",
      "Ep.: 716, Total Steps: 764165, Ep.Steps: 855, Score: 304.43, Avg.Score: 298.75, Time: 12:32:09\n",
      "Ep.: 717, Total Steps: 765020, Ep.Steps: 855, Score: 303.35, Avg.Score: 298.80, Time: 12:33:00\n",
      "Ep.: 718, Total Steps: 765899, Ep.Steps: 879, Score: 302.46, Avg.Score: 298.86, Time: 12:33:53\n",
      "Ep.: 719, Total Steps: 766730, Ep.Steps: 831, Score: 305.08, Avg.Score: 298.95, Time: 12:34:43\n",
      "Ep.: 720, Total Steps: 767589, Ep.Steps: 859, Score: 303.26, Avg.Score: 299.01, Time: 12:35:35\n",
      "Ep.: 721, Total Steps: 768429, Ep.Steps: 840, Score: 303.51, Avg.Score: 299.06, Time: 12:36:25\n",
      "Ep.: 722, Total Steps: 769284, Ep.Steps: 855, Score: 303.49, Avg.Score: 299.11, Time: 12:37:16\n",
      "Ep.: 723, Total Steps: 770131, Ep.Steps: 847, Score: 302.74, Avg.Score: 299.15, Time: 12:38:07\n",
      "Ep.: 724, Total Steps: 770950, Ep.Steps: 819, Score: 304.98, Avg.Score: 299.21, Time: 12:38:57\n",
      "Ep.: 725, Total Steps: 771797, Ep.Steps: 847, Score: 305.04, Avg.Score: 299.26, Time: 12:39:48\n",
      "Ep.: 726, Total Steps: 772644, Ep.Steps: 847, Score: 303.43, Avg.Score: 299.31, Time: 12:40:39\n",
      "Ep.: 727, Total Steps: 773498, Ep.Steps: 854, Score: 303.04, Avg.Score: 299.33, Time: 12:41:30\n",
      "Ep.: 728, Total Steps: 774359, Ep.Steps: 861, Score: 304.60, Avg.Score: 299.36, Time: 12:42:22\n",
      "Ep.: 729, Total Steps: 775201, Ep.Steps: 842, Score: 306.10, Avg.Score: 299.41, Time: 12:43:12\n",
      "Ep.: 730, Total Steps: 776073, Ep.Steps: 872, Score: 302.47, Avg.Score: 299.45, Time: 12:44:05\n",
      "Ep.: 731, Total Steps: 776931, Ep.Steps: 858, Score: 304.85, Avg.Score: 299.52, Time: 12:44:56\n",
      "Ep.: 732, Total Steps: 777775, Ep.Steps: 844, Score: 304.68, Avg.Score: 299.58, Time: 12:45:47\n",
      "Ep.: 733, Total Steps: 778594, Ep.Steps: 819, Score: 304.43, Avg.Score: 299.63, Time: 12:46:36\n",
      "Ep.: 734, Total Steps: 779420, Ep.Steps: 826, Score: 304.28, Avg.Score: 299.71, Time: 12:47:26\n",
      "Ep.: 735, Total Steps: 780240, Ep.Steps: 820, Score: 304.95, Avg.Score: 299.76, Time: 12:48:15\n",
      "Ep.: 736, Total Steps: 781070, Ep.Steps: 830, Score: 305.48, Avg.Score: 299.84, Time: 12:49:05\n",
      "Ep.: 737, Total Steps: 781887, Ep.Steps: 817, Score: 304.72, Avg.Score: 299.90, Time: 12:49:54\n",
      "Ep.: 738, Total Steps: 782729, Ep.Steps: 842, Score: 305.59, Avg.Score: 299.96, Time: 12:50:44\n",
      "Ep.: 739, Total Steps: 783560, Ep.Steps: 831, Score: 302.64, Avg.Score: 299.98, Time: 12:51:34\n",
      "Ep.: 740, Total Steps: 784391, Ep.Steps: 831, Score: 304.36, Avg.Score: 300.03, Time: 12:52:24\n",
      "Ep.: 741, Total Steps: 785221, Ep.Steps: 830, Score: 306.12, Avg.Score: 300.10, Time: 12:53:14\n",
      "Ep.: 742, Total Steps: 786048, Ep.Steps: 827, Score: 305.25, Avg.Score: 300.14, Time: 12:54:04\n",
      "Ep.: 743, Total Steps: 786900, Ep.Steps: 852, Score: 302.96, Avg.Score: 300.17, Time: 12:54:55\n",
      "Ep.: 744, Total Steps: 787736, Ep.Steps: 836, Score: 305.10, Avg.Score: 300.21, Time: 12:55:46\n",
      "Ep.: 745, Total Steps: 788543, Ep.Steps: 807, Score: 305.70, Avg.Score: 300.26, Time: 12:56:34\n",
      "Ep.: 746, Total Steps: 789357, Ep.Steps: 814, Score: 303.83, Avg.Score: 300.26, Time: 12:57:23\n",
      "Ep.: 747, Total Steps: 790168, Ep.Steps: 811, Score: 304.71, Avg.Score: 300.30, Time: 12:58:12\n",
      "Ep.: 748, Total Steps: 791038, Ep.Steps: 870, Score: 304.40, Avg.Score: 300.33, Time: 12:59:04\n",
      "Ep.: 749, Total Steps: 791890, Ep.Steps: 852, Score: 303.49, Avg.Score: 300.38, Time: 12:59:55\n",
      "Ep.: 750, Total Steps: 792712, Ep.Steps: 822, Score: 304.65, Avg.Score: 300.41, Time: 13:00:44\n",
      "Ep.: 751, Total Steps: 793539, Ep.Steps: 827, Score: 305.80, Avg.Score: 300.45, Time: 13:01:34\n",
      "Ep.: 752, Total Steps: 794376, Ep.Steps: 837, Score: 304.98, Avg.Score: 300.49, Time: 13:02:24\n",
      "Ep.: 753, Total Steps: 795215, Ep.Steps: 839, Score: 302.65, Avg.Score: 300.47, Time: 13:03:15\n",
      "Ep.: 754, Total Steps: 796075, Ep.Steps: 860, Score: 303.37, Avg.Score: 300.49, Time: 13:04:06\n",
      "Ep.: 755, Total Steps: 796883, Ep.Steps: 808, Score: 305.97, Avg.Score: 300.54, Time: 13:04:55\n",
      "Solved environment with Avg Score:   300.5364813490483\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from  collections  import deque\n",
    "\n",
    "def sac_train(max_steps):\n",
    "\n",
    "    total_numsteps = 0\n",
    "    updates = 0\n",
    "    num_episodes = 10001\n",
    "\n",
    "    time_start = time.time()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores_array = []\n",
    "    avg_scores_array = [] \n",
    "    \n",
    "    for i_episode in range(num_episodes): \n",
    "        episode_reward = 0\n",
    "        episode_steps = 0\n",
    "        done = False\n",
    "        state = env.reset()\n",
    "\n",
    "        for step in range(max_steps):    \n",
    "            if start_steps > total_numsteps:\n",
    "                action = env.action_space.sample()  # Sample random action\n",
    "            else:\n",
    "                action = agent.select_action(state)  # Sample action from policy\n",
    "\n",
    "            if len(memory) > batch_size:\n",
    "                \n",
    "                # Update parameters of all the networks\n",
    "                agent.update_parameters(memory, batch_size, updates)\n",
    "\n",
    "                updates += 1\n",
    "\n",
    "            next_state, reward, done, _ = env.step(action) # Step\n",
    "            episode_steps += 1\n",
    "            total_numsteps += 1\n",
    "            episode_reward += reward\n",
    "\n",
    "            mask = 1 if episode_steps == env._max_episode_steps else float(not done)\n",
    "\n",
    "            memory.push(state, action, reward, next_state, mask) # Append transition to memory\n",
    "\n",
    "            state = next_state\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "\n",
    "        scores_deque.append(episode_reward)\n",
    "        scores_array.append(episode_reward)        \n",
    "        avg_score = np.mean(scores_deque)\n",
    "        avg_scores_array.append(avg_score)\n",
    "        \n",
    "        if i_episode % 20 == 0 and i_episode > 0:\n",
    "            save(agent, 'dir_chk_lr0001', 'weights', str(i_episode))\n",
    "\n",
    "        s =  (int)(time.time() - time_start)\n",
    "            \n",
    "        print(\"Ep.: {}, Total Steps: {}, Ep.Steps: {}, Score: {:.2f}, Avg.Score: {:.2f}, Time: {:02}:{:02}:{:02}\".\\\n",
    "            format(i_episode, total_numsteps, episode_steps, episode_reward, avg_score, \\\n",
    "                  s//3600, s%3600//60, s%60))\n",
    "\n",
    "                    \n",
    "        if (avg_score > 300.5):\n",
    "            print('Solved environment with Avg Score:  ', avg_score)\n",
    "            break;\n",
    "            \n",
    "    return scores_array, avg_scores_array \n",
    "\n",
    "scores, avg_scores = sac_train(max_steps=max_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(agent, 'dir_chk_lr0001', 'weights', 'final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of scores:  756 , len of avg_scores:  756\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAEGCAYAAACtn3UnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3yb1fX/31eSZ7wdZznDWc7ei5EJARISEkjYlNUBhVIKbaGsAu2vfEmhlJY9StkQQoBQVgIJZLKyd0ic4cSJHY/E25Yt6fn9oWFtS7Zly9Z5v15+6dF97nOf+ziO7kfnnHuO0jQNQRAEQRCE9oKurScgCIIgCIIQDCJeBEEQBEFoV4h4EQRBEAShXSHiRRAEQRCEdoWIF0EQBEEQ2hWGtp5AKOncubOWlZXV1tMQBEFoV2zevLlY07SMtp6HIPiiQ4uXrKwsNm3a1NbTEARBaFcopXLbeg6C4A9xGwmCIAiC0K4Q8SIIgiAIQrtCxIsgCIIgCO2KDh3zIgiCIIQPmzdv7mIwGP4DDEe+PAu+sQC7TCbTL8eNG1forYOIF0EQBKFVMBgM/+nWrduQjIyM0zqdTgrrCV6xWCyqqKhoaEFBwX+Aed76iPIVBEEQWovhGRkZ5SJcBH/odDotIyOjDKuFznufVpyPIAiCENnoRLgIgWD7O/GpUcRtJAiCEGJq6szEResd7zVNY+uxUkZmJmPQe34+l1bXkRwXBYBSCpPZgkWDD7bkccmYTAw6xYHCSnRKse3Yacb1SSM+Wk96QjSbjpymrKYenYJZw7u32jMKQmvSZuJFKRULrAVibPNYqmnaQ0qpvsBiIA3YAlyraVqdUioGeAMYB5QAV2iadqRNJi8IQkRRUVuP0WShc0IMf/1kD33S47n+rCwKK2qJjzYQpVcYdDoOF1fSPyMBiwZf7TnJ1OzOrPmpiFve3sLK30+lZ2o8xnoL9y3byWc78jl7QDoje6YwqmcyU7Mz2Ha0lFvf2UJpdT2dovVU1ZmZlp3Bmv1Fjrnc++FOeqbGkXe6xu+cJ2aliXjxwZ/+9KduH3zwQbpOp9N0Oh3PPfdc7jnnnFPV1vMSAqctLS9G4BxN0yqVUlHAeqXUF8DvgSc1TVuslHoB+AXwvO31tKZpA5RSVwJ/B65oq8kLgtBx0DSNr/acJCU+mol90xxtdWYLJ0pr+dUbm8gprOTuWYP474bDAPx4+BSf7cx3jNE5IYbiSiMACTEGKo0mzuqfzrcHSwCY+c+1HvfdkFPChpwSr3OqqjMDuAgXO33S4ymsMFJnsri06zETQz0x1HHN8J7B/hoigpUrV3ZasWJFys6dO/fExcVp+fn5BqPRqJo6Xn19PVFRUS05RSEA2ky8aJqmAZW2t1G2Hw04B7ja1v468DBW8TLfdgywFHhGKaVs4wiCEIYcLakmMzUOva7xtaG8tp5X1h1meGYy07IziDZY3SkFZbV0SYxB52eMu5duZ2CXRH41tR+VRhNResUflmyne3Isv5zSj/RO0SzbdoL0TtHodIpp2dayPcWVRv637QTLth1nR14ZAJ0ToimurCPGoMPoJg4eW/6T43jl3pMMz0wit7iaCqPJIVwAKo0mMlPiHMLFmcyUOG48O4vqOjM78kqJ0uv4YlcBAIkxBh66oDeXdC3k8LE8iotOkhFVQ6y5CoOpCmN1OZ20GtKj6tCyKikrO0WCqgVjJQZTFZjrHPfR9k2EyV81+nuPNI4fPx6VlpZmiouL0wC6d+9uAlizZk38HXfc0bu6uloXHR2trV279qeYmBjtuuuu67Njx454vV7PY489duyiiy6qeOqpp9K/+OKLZKPRqKuurtZ9//33+//85z93/eijj9Lq6urUnDlzSp988skTbfukHZs2jXlRSumBzcAA4FngIFCqaZrJ1iUPyLQdZwLHADRNMymlyoB0oNhtzJuAmwB69+4d6kcQBMHGlqOn2Xj4FBeO6E5SbBRnLVrlsB6su3sGibEGHli2i/OHdaN/Ric2HTlNcaWRbsmxLNl4jPyyWgorjC5jDs9MYtfxcgB6p8Wj1ynOH9aVwnIjn+44wYUjutMzNY4lm/IA2HaslC/3FFBvbvhO8/K6w17n2z05lmiDjtySagCHWCmutAoAu3DJSIzhsYUjOXqqmme+yWHJzWfSLSnW6iqyxatk3fMZAHfPGkRqfDSXj+8FwN+X7+PsAZ05s186a/cX0S05luGZyR5zKauuZ/eJMs4a0BnevxG+/JABWD8YAVA6iE6w/sRYX1VMAik9sh3vHa+GWDDEopLC22V019LtvfYXVMS35JjZ3RKrH7901DF/fS6++OLyRx99tEdWVtbwyZMnl1911VWnzj333Kprrrmm/9tvv31w2rRp1adOndIlJCRY/va3v3UF2L9//56tW7fGXnjhhQMPHjy4C2DLli0JO3bs2N21a1fzhx9+mJSTkxO7Y8eOvZqmMXPmzAFffPFFwuzZsyv9zUVoOm0qXjRNMwOjlVIpwEfAEG/dbK/evnZ5WF00TXsJeAlg/PjxYpURhAApq64nIdbgYiXJL6vhma9zWLO/iMHdEtE0GN0rhYXjevLc6hxOVdURrdex9VipQwQ8+sU+4qP1VNuEC8CUx75xHH+6o8HV0hh24QJw9JR1/BfXHHK0fbztBKARQz1xGNm6s5h5veLo3knj+5/yiFN1xGEkljrHcRWxvG+eRk29GUP5Ud44vysTesYTrdWx40gRr63ZS5yqY/6wVMZ0jyHGUovK/Qbqq7l+aA2seh3qq8FYAbXlYKxga0wpf62/lkFdx3PukK6gaVBfzX1TUqH2NBQcZWZMOZSVQ2GF9VpjBdSWgbGCZGMFZxkrYE05nNgKQ+fD1LsgNgXiUqyiRDXZsyE4kZycbNm1a9ee5cuXJ65atSrx+uuv73/nnXfmd+nSpX7atGnVAGlpaRaAb7/9NuG3v/1tIcCYMWNqe/ToUbdz585YgClTppR37drVDLB8+fKktWvXJg0dOnQoQHV1tW7fvn2xIl5CR1jsNtI0rVQptRo4A0hRShls1peegN30lgf0AvKUUgYgGTjVFvMVhI6CpmmU15pQCkb99UtG9UrhkYuHs3jjUd76/qhLX3uA6Kp9hTzx1X6v42WmxHG8tMZFuBh0CpOl4XvEnTOz0etw7JDZV1BO/4wEBnVLBCAOIwU//cjqbfvoE1NNlLGUcRlmdDWnKCnKx1JZRLJWjrm2gljqMJjdAleLbD8xvp+7XOvEokvOJOXD36PWNriGxgBjom1v9tt+UBDdCaLibD+249gkSOgCMUlUbV3BX6New/DJ/+DjSqsw0Swe9/UgKh5iEm0/SdbX7Fkw7R7oOrTx69sxjVlIQonBYGDu3LkVc+fOrRg5cmTNCy+8kKGU8vZl2OcY8fHxFud+d9xxR/5dd91V7PMCoUVpy91GGUC9TbjEATOxBuF+A1yKdcfR9cDHtkv+Z3v/ne381xLvIgjBU15bT22dmS5Jsfzf53t5ed1hx5f67cdKmfv0eq/XzR/dg7kje1BaXcddS3c42heO7cns4d3okx7PwK6JrP6pkBW7C/h42wmq68zcPWsQU7MzOF1VT2qnKAZ3S3IZd1TPZKg+BSU74HQurHyYnqW5/My5U148xKfTJT4dunaH+OFW8RAVZxUAjp84pzbP1+W7jnPOl7N4MfpJ+OBJ0MfAZa9aRYMh1trXEAdRsQ1jGmIatXrc/+P/caX+a0Z36Ulcly7W8WKTXEWJ49VJrOjD4vtjRLF9+/YYnU7HiBEjjABbt26NGzhwYO3q1auT16xZEz9t2rTq06dP6xISEiyTJ0+ufOutt9LmzZtXsWPHjpj8/PzokSNH1v7www8u7q7Zs2eXP/zwwz1uuummU8nJyZbDhw9HRUdHa5mZmSbvsxCaS1v+z+kOvG6Le9EBSzRN+1QptQdYrJT6G7AVeMXW/xXgTaVUDlaLy5VtMWlBaA+crqpjW14p0wZaA1OVsuYL+ceKn3jmmxwAly24vr4GvHL9eI6UVLP+QBH/vnKMo71najyJsQav8RvT00qZ3msjw3d+zae6URh0QxmcApAPpblwONf6etr2WnoU6pys67oomPEADDwP4tOtP9EtExpRE6/xs7r76K0r5B9XjIf0/pA5rtnjrrGMYo1lFK+eOYHug7q0wEyFUFFeXq6//fbbe5eXl+v1er2WlZVlfP3113P3799ffPvtt/eura3VxcbGWtauXbv/7rvvLrz22mv7ZGdnD9Xr9bz44otH7IG+zixYsKB89+7dsRMmTBgMVqvM22+/fVjES+hQHdl4MX78eG3Tpk1tPQ1BaHGOl9aQW1LFWf07u7Sdrqoj2qDj1Q2HefdHV6u8PW+IMxOz0vjxyCk6J8Rw/rCuvPPDUX41pS/vbTzGO786w6s48YvFAv8cDJUnASjX4oiOjiW2/rRrv+gESOkDqX0gpbfTse01JjG4+wbIsq3HueO9bQAcWTSnxca1B+y+duMEpncA8aKU2qxp2viWHnf79u1HRo0aJa4VISC2b9/eedSoUVnezonNUhDaIXOeWkdpdT2bH5jJta/8yNAeSSzdnOc4nxTb8F/bntDMXbjs/9tsx3ZkOw/OHUpslJ775wQYb6FpUHYMTu6Gk7vgxDarcJnzT37zxWl+blpMYpfhZA8Z7SRUsiA+rU0CUEN9SyVBtYLQKoh4EYQw5Yud+aTER5MUZ2Bo9yTWHSjmqVUH+PdVYyitrgdg2uOrqTSa2JNf7nJtea2JRxeM4Mx+6WR17sSyrcfpFGPgV29YLZE9U+M8hAtAbJTeo82FqhI49A0c/c4mWHaD0eneqX1h2CUwfCE/rNjIZ3VD+MuwYWSfldWs34UgCIIzIl4EIQx59Iu9LluCnTl70deO40pjg0t9fJ9UiiqNXDOpN+Oz0hjbO9Vx7uIx1nRJ9184hEc+3xvYJDQNivbBkfVwfIt1C2/RPkCD6EToNhxGXg5dh0HX4dBliFd3TyQZIyLoUQWhTRHxIghtxJHiKtblFLP4x6NMGZjBhKxUeqTEkd010adw8cc1Z/TmkjH+U8I3KiQqTsKh1VbryqHVUGHLyRLfGTLHwvAF0P9c6DEadI1Yaez3DKhX6xBqt04kCTVBaEtEvAhCG3HDqz9yxJbYbfeJcl5YY22/aqLvzNDzR/fg420nmD4og3qzhQ05JShlNZLEGhoXEz4X7xNbYdlvoHC39X1cGvSbDv1nQN+p1kDapq7MYbSih3omKqykmiB0XES8CEIr8eqGw4zsmcy4PmnsOVHuEC7uvPtjQ3K4Id2T2Jtfzk1T+3HhiO68v6lhB9G9s4dw/7JdXDy6B3/5ZI8jyZs/XMoDWSxw9FvY9QHs+tCaz2Tmw9BvBnQbCTrPmJimEE7LeegDdkM7viAIVlrm00kQBL9omsZfPtnDwue/A2DzUevW4V5pcbx30xler7n9nAF8ctvZ7Pt/s7jvwiGM7pXiWBwVMDwzmY9/czY3nJXFzofPp19GQqPz0AGJVGOw1MPHt8Jrc2D7YhhwLvzsA5h8p80l1HIfDZG0oEfQo7Zr3njjjRSl1LitW7fGtvVcAAoKCvSTJk3Kjo+PH3Pddde5mF7XrVsXn52dPbR3797Db7jhhl4WizWx78mTJ/VnnXXWwD59+gw/66yzBhYVFQXmx22EadOmDSguLm7WWJ9++mnijBkzBjTes+mIeBGEEGIyW/j2YLHHNuU9J8qINuhY88cZTOqXzvf3nku3pIbP0QfmDOF3M7Mx6HVedwA5u3+UUiTGRgU0n7E5T7Ez9pesNF4F29+FiTfDXTlw6X+h24gmPqV/wsmVEvK5hM+jCn5YvHhx2tixYyvffPPNtLaeC0B8fLz217/+9cTDDz+c537u1ltv7fPcc8/lHjlyZNehQ4dily5dmgTw0EMPdZ8+fXpFbm7urunTp1c8+OCD3VpiLmvWrMnp3LmzufGebYuIF0FohENFldSbA6hT44W/fLKHq1/+gVfXN1Q2zrrnM9798RgZCTHobH6cbsmxmJ0SRl57Zh+XAol2mrX4rnuCEYf/C8By/TS4+n2YtchatyeEhJPlJeRuI1EvYU9ZWZlu06ZNCa+++uqRjz76yLElb86cOf3ee+89R1bGhQsXZr322mspFRUVugsvvLBfdnb20Dlz5vQbOXLk4LVr13qkfP74448ThwwZMjQ7O3voZZddllVTU6MAMjMzR9x55509hg4dOiQ7O3uoN2tPUlKS5YILLqiMjY11+aDJzc2Nqqys1M2cObNKp9NxzTXXlCxbtiwVYPny5Sk333xzCcDNN99c8sUXX6S6j2symbj55pt7Dh8+fEh2dvbQxx9/vDNYLSPjx48fdN555/Xv37//sKuvvrq32WzVK5mZmSPy8/MN5eXluunTpw8YNGjQ0IEDBw57+eWXU/0959KlS5P69u07bNy4cYOWLl2aYp9DeXm57rLLLssaPnz4kCFDhgx96623UgA2bdoUO2LEiCGDBw8emp2dPXTnzp1+qpF5IjEvguCHogoj5zyxhp+d0Zu/Xexqmfhoax4T+6aTmRLn0m6xaLzx3REuHd+LD7dYv0h5K2Q4e7jrFyWzrXjhrGHdiPERfOvsNgqaLW9QkDaBs0/cTreUBOZmn9OUUYImnJbzkAfshtPDhjvLftOLwj0tU/fBTpeh1Vz8rN+Cj2+//XbK9OnTy0aOHGlMSUkxr1+/Pn7y5MnVV1xxxan33nsv9Yorriirra1VGzZsSHr99ddzH3vssS4pKSnm/fv379m4cWPsmWeeOcx9zOrqanXzzTf3/fLLL38aOXKk8ZJLLsl6/PHHMx588MFCgM6dO5v27Nmzd9GiRRmLFi3q+t577+UG8ji5ublR3bt3r7e/79OnT11+fn4UQElJiaFPnz71tvb6U6dOeazn//rXvzonJyebd+3atbempkZNmDBh8EUXXVQOsHPnzk5bt27dlZ2dXTd16tSBb7zxRuqNN97oSIX94YcfJnXr1q1+9erVObb76X095x//+Mei2267Leurr776adiwYca5c+f2s49z3333dZ8xY0b5+++/f6S4uFg/fvz4IfPmzSt/+umnM2699daTt9xyy6na2lplMgVXSUEsL4Lgh41HrIXL3VPtm8wW7nxvO2cv+pqjboG393y4g4c/2cMTX/7k4S5yZnyW6xelRy4eTmZKHE9fPcbHFQ2Lb9CL5OlcOH2EvK7nYqZFXOMBIwu6EE4sWbIk7aqrrjoNsHDhwlN219Gll15a9u233ybV1NSopUuXJk+cOLEiISFB+/bbbxOuuuqqUwATJkyozc7O9oi03759e2zPnj2NI0eONALccMMNJevXr3dE0F999dWnASZOnFh97NixgC0M3sr3BLPdf+XKlUlLlixJHzx48NAxY8YMOX36tGHPnj2xACNGjKgaOnRoncFg4PLLLz+1bt06l6C5sWPH1qxbty7plltuyVy+fHlCenq62ddzbtu2LbZnz57GESNGGO0WIvs4q1evTnryySe7Dx48eOjkyZMHGY1GlZOTE33mmWdWPfHEE93vv//+bgcOHIhOSEgIqlaRWF4EwQ+Hi6sA6Jro+nnj7OKZ+vg3bH5gJukJMRhNZpZsslpbth4t9RjPoFOYbBaW+GjX/36zR3Rn9ojuAc4sSEXww4sAnOx8BuB9l1OoCCdXSujdRkLANGIhCQUFBQX677//Pmn//v1xt912G2azWSmltOeffz4vPj5eO+OMMyo+/PDDpPfeey/VLlgCqf/XWJ/Y2FgNwGAwaCaTKeA/k6ysrHq7pQUgNzc3ulu3bvUA6enpptzc3Kg+ffrU5+bmRqWlpXmYLjRNU0888cTRhQsXuqTg/vTTTxPdRZD7+5EjRxq3bNmy54MPPki+//77M1euXFm+YMECzw81H9c7zYGlS5fmjBo1yujcPnbs2NopU6ZUffTRR8mzZ8/Ofu65547Mmzevwucvww2xvAiCH+wfSjq3+BP3z6qSqjoAjp+ucbRtO+b5//zT2yc7juOjg7eANCnJmrESNr4MWVOoTOzXeP+WJqxW9FAnqQurhxXcePPNN1MXLFhQcuLEiZ3Hjx/fWVBQsKNnz551X375ZQLAlVdeeeq1117rvHHjxsQFCxaUA5x11lmVixcvTgXYvHlz7P79++Pcxx09enTt8ePHo3ft2hUD8MYbb6RPmTIl4IXYF3369Knv1KmTZdWqVZ0sFgtvv/12+vz580sBLrjggtIXX3wxHeDFF19MnzVrlscHznnnnVf2/PPPZxiNRgWwY8eOmPLych1Y3Ub79u2LNpvNLF26NM19vkeOHIlKTEy03HrrrafuuOOOk9u2bYv39ZyjR4+uzcvLi969e3cMWAOi7ePMmDGj/Iknnuhq3yW1YcOGOIA9e/ZEDxkyxPjAAw8Unn/++aXbtm3z+L36QywvguCFY6eqee3bIyzbetzreXfxYq81dPSUf6tGlL7h+0JcE8SLnaDWyH2fgbkOpt2NKm797yvhtJxLnpfI5v3330+/++67853b5s+ff/rNN99MmzVrVuUll1xS/utf/7rvzJkzS+3Wkrvuuqvo8ssvz8rOzh46fPjw6kGDBtWkpqa6+IPj4+O1F1544chll13W32w2M2rUqOo//vGPRcHMLTMzc0RlZaW+vr5erVixIuXzzz/fP27cuNrnnnsu9xe/+EXf2tpaNWPGjPLLLrusDOAvf/lL/iWXXNK/T58+nXv06FG3bNmyg+5j3nnnncVHjhyJGTFixBBN01RaWlr9559/fhBg9OjRlX/4wx967tu3L27SpEkV1157rYv42bx5c9y9997bU6fTYTAYtOeeey7X13PGxcVpTz/9dO7cuXMHpKWlmSZNmlS5d+/eOIBFixaduOmmm3oPHjx4qKZpqmfPnsZvvvkm580330x7//330w0Gg5aRkVH/6KOPngjm9yXiRRC88NHW47zitEPIHQ1X9VJabbW8FFUYvXV3oHNa3dzdRoEQdMDuR7fA9ncgfSD0mYwq8S7GQkkkWSMi50nbJz/++ONP7m0PPPBAof04JiZGKy0t3eZ8Pj4+3vLhhx8ejo+P13bv3h1z/vnnZw8cOLDOfZz58+dXzJ8/f497+/Hjx3faj6dOnVrtbQ7u/ZyZOnVq9YEDB3a7t3fr1s383Xffee4EcEKv1/PMM88cBzz+48fFxVk+++wzjzok9nksXLiwfOHChR7P4+s5L7300vJLL73UY54JCQnaO++84xGg/OijjxY8+uijBf7m7w8RL4LgRqXRxDNf5/jtY3G3vNTU29qtJy4a1YNPtnt+kXBe3JriNnKME8gqeXyLVbiMvALO+3+g07WJkAinBV12GwnBUlFRoZsyZcqg+vp6pWkaTz75ZK7dKiO0HSJeBMGNBz/eRV0jeV3cA/Tslhe7qBnVM9mreHG2vDTFbRRw8GvhXvjPTNBFWbPmJna13T/oWzabcFrQQy/ewuhhhRYhNTXVsmvXrgBLsbcP5s6dWzF37txmx+S0JRKwKwhurD9Q7NHmHuPibnmpNJpd+nlLMAfWhTw5zrp5IN5L5tzGaHAbNbJI/viStVbRLRugyxBHs64tLC9htJ6L5aXNsVgsFvktCY1i+zvx+S1SLC+C4Eahl7gVd0uLx9ZI23u728jgQ7zodIqPbj2Lbw+WYNAH/90hoDwvmgb7V0D/cyBjkOv1bWF5EWuE0MCuoqKioRkZGWU6nU5cL4JXLBaLKioqSgZ2+eoj4kUQAsD9U9Zdu7z9w1H6d0nwubXajk5Bv4yEgIoo+sOvCDm5C8qPw/R7vVwX4ZYXyfPSpphMpl8WFBT8p6CgYDhi+Rd8YwF2mUymX/rqIOJFEALA023k2lBSVcfvFm/jL/OsmcP1TqvkxvtnMuGRlUDzrRABLb77l1tfB57veX2z7t7+Cf1W6Uj/Dftn3LhxhcC8tp6H0P4R5StEPIUVtRSU1QJQZfReX8NdrPiyd9stL84xL85GmOYGzNoXR78iaP8K6DHGEaTrTNvEvITPgh5qF1b4PKkgdGxEvAgRz8RHVnHGo6sA3+LFXay4i5mGduurq3hpOG7uQq48DtyoKoa8TZA9y/v1bRLzEjmEkU4ThA6NiBdBcMJ9F5EdzwBdX9d7s7x4t8I0B5/DHPjKOrnsC7yejvSt0qFWUhKcLAitg4gXQXDCl0Wlsa3Sdo7ZygO4bJV2PmzuSt7Y5TkroVMX6DaqiQO0POG0oMtWaUHoGIh4EQQnfMayeLz33vP176xZsA2hinmxLb9eRZDFAodWQ7/poPP+XzvSLS/hFH8jCELTEfEiCE5YfJhU3C0yviwvdlxdRS0X8+IYx1tj4W6oLraKF1/XRXh5gFAj2kgQWgcRL4JgI6ewktn/Xuf1nIfbqBH1YtCHJubF7+J4aLX1td90n10i3vLS1hMQBKFFEPEiRDTOIuT/Pt9LZYBbpRtD7+S2cV68W2q3kddhDq2GztmQnOn7+jZZvcNHMoQ+SV34PKsgdGREvAgRTa3J7Dj+el+h746NJKlzR+/DbdRSlhePYUxGOLIB+s1o5PoIz7Ab6jwvYfSsgtCREfEiRDTbj5UF1K+x8gDuOMfLugbstlDMi/s4x34EU41fl1FL3j8Ywmk9D32G3dCOLwiCFREvQkRz1cvfB9TPM2C3kZgXJ/XiGrAbxOS84NNycGg1KD1knd3I9a1PJO3wEbeRILQOIl4EIQACzFHnwLlgtGpBy4tPt9Ghb6DneIhN9nt9xFteQj1+OD2sIHRgRLwIQgB41DZqNIDX+/bo5q5tHuUBSg7C/34Lxzf7LAngcn2E7zYKfYZdQRBagzYTL0qpXkqpb5RSe5VSu5VSv7O1pymlvlJKHbC9ptralVLqKaVUjlJqh1JqbFvNXegYNC5AnPp6XOu/v6/A3JayfBgs9bBzKSy+2vo65CKY4LN6vINIFy8SsCsIHYO2tLyYgD9omjYEOAP4jVJqKHAPsErTtIHAKtt7gNnAQNvPTcDzrT9loSNhsm2THtXTv6sF8LLbyH93XyKl2YubbYA5hS/CB7+AykJY8DJc8RbEJjV+eYSXBwg9kfSsgtB2GNrqxkxeGvcAACAASURBVJqm5QP5tuMKpdReIBOYD0y3dXsdWA38ydb+hmb9uvy9UipFKdXdNo4gBE292QJAtKFxDR9swK4vkdKs4NXacmbv/zPXxnxHQqkJsmfDlW+DTh/wEG2RpC6c1nOxjAhCxyAsYl6UUlnAGOAHoKtdkNheu9i6ZQLHnC7Ls7W5j3WTUmqTUmpTUVFRKKcttHPqzVYBEqVv/L9B8G6jEKySP77IkKLlZKgyyg3pMO3uoIQLSHkACdgVhI5Bm4sXpVQC8AFwh6Zp5f66emnzWEI0TXtJ07TxmqaNz8jIaKlpCh0QUxtYXppM/g7Y+AoFCcMYU/sCjw94EzKDD/tqm/IA4bOih3ou4fOkgtCxaVPxopSKwipc3tY07UNb80mlVHfb+e6APe1pHtDL6fKewInWmqvQ8bBbXqIDsbwEVx2gZS0v9bXwye9As/D1gHs5TVKT1VGbBOy2/i3bjHASaoLQkWnL3UYKeAXYq2naP51O/Q+43nZ8PfCxU/t1tl1HZwBlEu8iNIdgYl7AdXdSq1le8jbBi1PhxBY4588UJw5u1nARXx5AtkoLQoegLS0vZwPXAucopbbZfi4EFgHnKaUOAOfZ3gN8DhwCcoCXgVvbYM5CB8K+2ygQywu4Wl9aJebl+Gb47yyoq4RrPoCx1zZ7cWybJHXhs6RLzIsgdAzacrfRenx/lpzrpb8G/CakkxIiimBiXsA1wKoxy0uLxJZsfRv0UfDr9RCf1gIDtlV5gDa4qQ+kqrQgdAzaPGBXENqKuiDFi7NgaTwEppmLmMUCB76CrCkuwqW5i2+klwcINeEk1AShIyPiRYhYTEEE7IK72yjElpe9H0PZURi+wKW5uTErbbK4htWCHlaTEQShiYh4ESIWk8VqeYkK2G3kHLDrv2+zLBxlefDJHdBjLAxf2PRxvNA2u43CRzCIZUQQOgYiXoSIJZit0tBKAbsWM3x4E1hMcOkr1piXFqRNygOEkWCQgF1B6Bi0WcCuILQ1wW+VbjgO2VbpDf+C3A1wyYuQ1q/lxrWha4OvK5G0nkueF0FoHcTyIkQs9piXmKYE7DZieWnSGlZ6FNY8BkPnw6grvY/bTCnQNpaX8FnQJcOuIHQMRLwIEUvQlhfn40YDdpuwjH37NGgWOP+R4K8NkLYpD9D69/SFuI0EoWMg4kWIWOxJ6gIpzAiugqWxrdJBL2J11bD9PRh6MaT08tmtuYtjpJcHkDwvgtAxEPEiRCzBihdLEDEvQVtetr4JxjIYf6Pfbs1dGiO9PECoiaRnFYS2RMSLELHYLSkBu1KC2G0U1CJmMsLaf0CfydD7zJYb1wttkaQunGwvobaMhM+TCkLHRsSLEJHkl9WwIacYAH2A6sUSTGHGYJax/SugqhAm3xHyr+5SHqCtZyAIQksgW6WFiOSCJ9dSXmsCAneluAbs+u8bVGDsziXQqQv0m9Fo1+ZaDqQ8QIiJqIcVhLZDLC9CRGIXLgD6QMWLS8BuC8W81Jy2Wl6GLwR9498l2mXAbgSZOyRgVxBaBxEvQsQTqJXEJWDX0tiYAQ667V0w18GoKwLr3w4Jp+U85LuNwulhBaEDI+JFiCiKKoxk3fOZS5suQPXibG1ptKp0IEOa6+H756D3WdBjTEBzaI+E04IuAbuC0DEQ8SJEFLuOl3m0BWolCaY8QEB6aPcyKDsGZ/8uoPtD+3TBhJMrJfSWl/B5VkHoyIh4ESIKbzldAkzz0rKFGTUNNvwbOg+CgecHNgHa5zf7cFrPQ5+kThCE1kDEixBRVBpNHm0BW16c3UbNLcx46Bs4uRPO+m1Q1RLDSQgInsi/jyC0DiJehIiiorbeoy1Q8eIcsNtYzEujY274NyR0g5GXB3Tv9kw4Lejh5MISBKHpiHgRIgpvlpdAk9RpwSSp8zfkyT1waDVMuhkMMQHd2zFuUL3Dg3ASDFLbSBA6BiJehIiiotZTvAS6oLkG7Prv69fysum/oI+BsdcHdmMnmhsQ2jZ5Xlr/nr4I+VTC6FkFoSMj4kWIKLy5jQJPUud83Fh5AB9YLLDzfRhyEXRKD+i+LuO2w8WxPc65qUTSswpCWyLiRYgoSqrqPNqalOelqZaXwt1QWwoDzwvonh2BcHKlyG4jQegYiHgRIoqiCiOjeiZzzaTejramBew2MeZlyxugM0DfaQHd02PcJl3VQKdoawmCcX1SmzlS4ISXNSLESerC62EFocMihRmFiKKowkivtHh6p8U72gItD+ASsNtIeQBvi1gylbD5dRh1FSR1D+ymngM37TobqZ2i+eJ3U+jbuVOzxgmGcFrOxfIiCB0DsbwIEUVhhZGMxBgXa0ugu42C2SrtjTN0e8FshNHXNOHqlmNI9yRio/Stdr9IMkZE0rMKQlsi4kWIKMpr6kmOi3JZZAIuokjgW6Xd+e05A/h5r3wwxELm2KCudaZ9ro3hM+tQzySc4nsEoSMj4kWIGMwWDZNFI8agc3HrNKW2UbCmlz+cP4hJun3Qc0LQuV2caY/f7MNpzhKTIggdAxEvQsRQZ7IGqkQbdC5xLoFm57cEUZjRg9pyKNgBvc8M7jo32uM3+3CaccgtL+H0sILQgRHxIkQMdvESY9C7xrw0obaRvyR1XZO8WFaO/QiaBfqcFdhkOxBi7RAEoaUR8SJEDEazGfBmeQkwYNdph5GvrdKT+qax4U/nON7fNLWf9eDot9Yt0r0mBjdpN9qjDginKYd8t1E4PawgdGBkq7QQMRjrbZYXvQ6juUGJNKWqtC/LS5Reh0Fv/U5wZNGchhMHvoQeYyC6eVuU2+PaGE4Leqjdbu3RrScI7RGxvAgRQ53Ze8xLU8oDNJpi15mi/VCwE0Y0v4J0OAmBQAmnBV0sL4LQMRDxIkQMDTEvOhdrS0sWZvQ61uE11tcIKgngTCQt6BH0qILQprSpeFFK/VcpVaiU2uXUlqaU+kopdcD2mmprV0qpp5RSOUqpHUqppifLECISX7uNAk1S51rbKAjLy5H1kNQTUrMCv8YH4WTFEDyR4GRBaB3a2vLyGjDLre0eYJWmaQOBVbb3ALOBgbafm4DnW2mOQgfB2W3UlDwvloAsL25jaZpVvPSd0jImiHa4NobTei7lAQShYxCweFFKTVZK3Wg7zlBK9W3uzTVNWwuccmueD7xuO34duNip/Q3NyvdAilKqiQVihEjEHrAbrde5LDKB5nlxtrb4srt4LF5F+6C6GLImBzrNDkc4WSPCaS6CIDSdgD62lVIPAX8C7rU1RQFvhWhOXTVNywewvXaxtWcCx5z65dna3Od6k1Jqk1JqU1FRUYimKLQ3jhRXsWJ3AQAxUa55XgLfbeR07MNt5DHUkfXW10gWL209gVZEtJEgtA6BbpW+BBgDbAHQNO2EUioxZLPyjrePBY8VRNO0l4CXAMaPH9+U+nlCB2T6P1Y7jqP1OhdrS+C7jRqvbeQx0qHVkNwLUvoENtEOSDgt6KHPsBtGDysIHZhA3UZ1mvWTWwNQSjUvWYV/TtrdQbbXQlt7HtDLqV9P4EQI5yG0c06U1nhtj3bbbdSU2ka+4nVdFq/KQti/HIZcFF4reCsTTkHGEfzPIAgdikDFyxKl1ItY40x+BawEXg7RnP4HXG87vh742Kn9OtuuozOAMrt7SRDc2Zx7irMWfc1HW/M8zsUYXP/sm1bbyHsfl7Xx8FqwmGDEZYHdoIMSToIhnISUIAhNJyC3kaZp/1BKnQeUA4OABzVN+6q5N1dKvQtMBzorpfKAh4BFWMXSL4CjgP2T/3PgQiAHqAZubO79hY7LwcIqANYfKOGSMT1dzrnneQnc8uIcsBtAzMv+FRCTBN1GBjjrjonIBUEQWppGxYtSSg+s0DRtJtBsweKMpmlX+Th1rpe+GvCblry/0PH4v8/38v2hEn4x2boZrrbe7NHH3W0UeJ4Xp2Of0VS2sQ6thp1LYPwvQB/hVTjCSL2EkxVIEISm06jBXNM0M1CtlEpuhfkIQrN4ae0hduSVUW+2qgvf4qXhvbvl5fZzBjiOH5gzhJunWYsrOgfpNrrbaM/HoI+BmQ8H/xAdjHBy1YTPTARBaA6BxrzUAjuVUq/Ystw+pZR6KpQTE4TmUFlbD8D6nGLqnYowgi3Pi5/yAL+0V4IGfjmlHzMG2XbrBxPzcngt9JsOsUnBT14IHaJeBKFDEKg9+zPbjyC0CyqNJgCMJgubc0872nUKDHqdi2BpbD2zW2YCcRspBZw+AiU5ME7CssKNcLICCYLQdAIN2H1dKRUNZNuaftI0rT500xKE5lFhEy8APxxqSOIcpbcaG10LM7ouaO7Lm/20JaA8Lwo2/gd0Bhg6rylTFwRBEBoh0Ay704EDwLPAc8B+pdTUEM5LEHxyuLiKV9Yf9tunotZEeqdodAoqaht0tsEW7KLzY3lxFzP2vq55XvzEvOz/EvpOhZTefufYJCTtYrOQgF1B6BgE6jZ6Ajhf07SfAJRS2cC7wLhQTUwQfHHFi99RWGHkqom9iI/2/idcXGEkPSGa8tp6l5gXvUO8+I558VzfrC2WAGobpZlOQvFPMO56Hz2EtkS0iyB0DAIN2I2yCxcATdP2Y61vJAitTkWtqdE+BeW1dEuOQ6eUo5o0NLiNnFcxD7eRu5ixW16c2ny5jSZVrLQeDJjZ6BybhKy+zULS9wtCxyBQ8bLJttNouu3nZWBzKCcmCM1hR14ZXRNjMOgURlODeDHoPS0v7rgHdeq8qBdv2mWAymNuyavQ/1zIGNT0yQuCIAh+CdRtdAvWBHG3Y/3utxZr7IsgtCrPfpNDjZfcLd5IjotCp1OOnC8ABp09YDfwe9q7ugbsuvYZqPL4teF/6LDAvBBmEZCYl2YhdhdB6BgEankxAP/WNG2BpmmXAE8B+tBNSxAaqDdbuOmNTezNL+fxFQ7vJXvzK/xed/1ZWeh1ijpTg9jxFvPijvspx1ZpZ8uLk4q4Qb+cr2LuZqF+PXviJ0CyazkCIXwQr5EgdAwCFS+rgDin93FYizMKQsj5qaCCL/ec5A9Ltru0L3z+W77ac9LndUlxURh0ijovbqNgFjFvW6Xth1kqn4ej3gDgoKU7S7r+PvCBm4Isvs1C8rwIQscgUPESq2lapf2N7Tg+NFMSBFdio6xGPm+p/vef9G190Smr1cTVbWQTL34WMV/CxjVJnfXd5fo1mDXFxNpnObfuCU7HdPc5bqRx49nW+lKJsRFe20kQhBYnUPFSpZQaa3+jlBoP1IRmSoLgil1wFFcaPc59d7DEkU3X7BaIotcpm9uowfJidwH5i3nxFbCruZUHGK/2cavhf3xjGU0hqbZrBTu/mTGAI4vmOMRnWCD/QILQIQhUvNwBvK+UWqeUWgssBm4L3bSESOfYqWp++fpGqutMmG2qodzLFun1OcXcsXgrgEcNI52yiRezF/HiR7343CrtHLBrsfCvaGvM+lvm85z6yuoYzsg/jyB0DPzac5VSE4BjmqZtVEoNBm4GFgDLAf8pTgWhGfzf53tZubeQb/YVkd01wW9fe+Cu+/Zlh3hxtrzo7Odc+ybEGBwWHHesgkcj6dR2eOs2qC3jluJiuqhiFtVfyWrLaEdfWRvDG/n3EYSOQWOWlxeBOtvxmcB9WEsEnAZeCuG8hAjHLkSUApOvEs7u17jtI9Yp0CvlmmHX8dXbdRnb9ZcLHMfuC5yhppjF0X/j7G+ugJyVkLeRLrWHydfSeMN8vmtnWR0FQRBCTmORdHpN0+xV7a4AXtI07QPgA6XUttBOTYhUNE3DZLEKDp3yjGXx1t/66truiHlxEi8qkJgXN99C6o6X6a/by96hv2fIGbMgOZOnlm/nra2nqSbW9VpRL2GNuPUEoWPQqHhRShk0TTMB5wI3BXGtIDSJP32wg5V7CwHrYuMrFb877v2U3W1kdI55sb/6y7DrhMlI8p63+cI8gfpBv2JI7x4AFMacptDbtbI2hjXyzyMIHYPG3EbvAmuUUh9j3V20DkApNQAoC/HchA7MpiOnuOS5DVR5iTNZsinPcaxo3PJi/zbt3Ms5GV29t4DdQJPU5W1CbyzlQ/MUl4BdX3pKFsfwRsSlIHQM/IoXTdMeAf4AvAZM1ho+vXXAb0M7NaEjc/u7W9l6tJStR0v99gvE8uJwGzltNrJbWPRutY3su4z8LWIuroWcr9CUnh8sQzy2Snu/1u9UBUEQhBag0a3SmqZ9r2naR5qmVTm17dc0bUtopyZEAvsKysm65zMeX7HP63kFmMzBB+zaLSt6nbvlxTZuACKjOyWw6b/U9JpKOZ3cRJQUGWqPSEySIHQMAs3zIggtSnyMNWTqdLV1M9uz3xz02s+saY48L43h3M1ZvHhPUtf4InaDYTnUVXF6ysMe41ss3q+RxTG8EcuYIHQMJOhWaFVKKo18ueekI9V/Y1YVk1nzKRTccbaM2GNe9Eq5uHgCFS8x1HG5fg0MnoMlPRs47mJr8eXKksVREAQh9Ih4EVqVRz7fy4dbjjveN5bDxWSxBG55cTpWTjEveGn3t1Ua4CL9d6SqSpjwS0ebS2FGH9eJeIlMYgw6l9gqQRBCi4gXoVWpcEvxb3KKR6k3W/jrJ3vczmtYAkxS59Xy4qZS9AEE7HJkPf+IepHDlq70zZqCrqzW2u4SsOtTvgQ0V6FtCJW4XHv3DIoqPGtvCYIQGiTmRWhV3D/g65zcRqt/KuLN73NdzpssloAz7OLNPeQmXuztfpOVrfsnAA+abgTVEMXiIlhkt1G7JFQxSV2TYhmemRySsQVB8ETEi9CquOd1cd4JpHmxZtSbtUbzvDiudzq2ixSDh3hxPe/OYHUUDq7iH/WXsc4yEnAqzOjUz2fMS0AzFdoKEZeC0DEQ8SK0Ku6Vn93fu7NidwHVdd4LJrrjLCh8iRS7xUXvZRXTYeHZqH9DVCfeNp/rNJYtCV7jhpfQL46yQ1sQBEHEi9Dy/GfdIaY+9g0nSms8ztW77S5qTLysO1DM3Ut3BHRf71ulXfvole+Yl4t039Jflw8XPs5pkhzt3txGPpPUie0lrJF/HUHoGIh4EVqcvy/fx9FT1RworPQ45y5W6gLYoRF4VekG7JYXg871T9z+1j2Ql+ID/Dv6Oaq1GBhxqcspb+UHvLm4rH0DmmrTkdW3WUhhRkHoGIh4EVqUHw+fclhXvNUt8hAvAWbP9Yd9BOddScpHwK7yledl7T8AuLP+FjDEuF1ju4/UNmr3yL+PIHQMZKu00Gw0TeNgURU6BZe/+J2jvbLWm3jRUKph8a93sry0ZDiHI8+L22rVsAvJqXH3MtixmNdN57HCMtFzLPv8AtgqLd/sBUEQQo9YXoRm89HW48z85xrOeWKNS3uFD8tLrEHv8t4bC8ZmBnx/bzEpdg1hTxyWmRIHwNyR3QE3y8t3z0BqFn8zXet1/IaAXev4O/JK+WJXQcDzE8IH0ZaC0DEQ8SI0C5PZwu+XbHdpu3vWIMDqNlqy6RhZ93xGTZ21HEC92UKf9HhHX1/ipUtibMBzsEsWbwG7m3NPA/D3hSM5smgOFwzrBjQE7o5QhyBvI0y6hXofhkj7gmf3Ss17ZoPPucjiGN6IZUwQOgbtTrwopWYppX5SSuUope5p6/lEOidKaz3auibGEmPQUWk08fTXBwCY/+x6bnlrMxYNZg/vzqs3TmBcn1SfMS+Npe/3hkt5APur7WB8Vqrr+LYb3GBYAdEJMPpqn+N6C9j12VeiKgRBEEJOu4p5UUrpgWeB84A8YKNS6n+apu3xf6UQKurMZo+22Cg9ibEGdp8oI8q2V3n/yUr2n7TuPooyKGYM6sLLaw9RVlPvddymfEF2zfNiHeCdX53B0ZJqYqP0Ln11Crpyiot038Kon0NsEr7wFrDbWF9BEAQhdLQ3y8tEIEfTtEOaptUBi4H5bTyniKamztXtM7Z3CrOGdyOtUzQbckoo9yJOom2CRq9TLlulndf9xqo+e0PzYnrpn5HAjMFdPOew8x1+iL2NaGWGs27zO663gN3G+gqCIAiho72Jl0zgmNP7PFub0EbUmlwtL1dN7I1ep/jbxSMAKK6s87jGbo0x6BRHT1U72t21x8rfT2PeqB4Bz8XZMuJXRBTtJ/rLe6nUYvl7/ZWQmuV3XEfAbgCOI7G8CIIghJ72Jl68LQ0uK4pS6ial1Cal1KaioqJWmlbkYg/EtWNf6Mf3SSUx1rtX0mDbv5xbUu31PFjjTAZ0SeC+C4cEPBcX8eNLRdRVw/vXQ1QsM42P87x5XqPjugfs+u8r6kUQBCHUtDfxkgf0cnrfEzjh3EHTtJc0TRuvadr4jIyMVp1cR2dHXinbj5W6tNXUu4kXnf1VcXb/zl7HsVteiiuNXs9Dg2DomhTjs487rruNfHT64m4o3INl/gsUkB7QuN5qG/lCpIsgCELoaW/iZSMwUCnVVykVDVwJ/K+N5xQxzHtmA/Ofdd0mXOsuXpwsD09dNcblXIzB+udmj3mpqvMM9nUfJxhLhkueF3cZYTHD14/A1jdhyh/QZZ8X8LjexveJqBdBEISQ0652G2maZlJK3QasAPTAfzVN293G04po3N1GzmIj2qDjthkDyEyN48IR3Tl70dcYTRaH5cXsxw/TFA3grC1cNE/FSVj3D/jxJRhxOUy/LyhRFIwnSLZKC4IghJ52JV4ANE37HPi8recRyZRV15McHwV4s7y49v3jBYMcx1G2WBeDe85+LzgLhj+en80/vtzvo6dzpWe32kY1pfDOFXDse2vjGb+BCx4JOqrWPcNuoPMWBEEQQkO7Ey9C27PwhW9Z+ftpANTUu26V9rfF2b70Rxu8eytdLScN49x2zkCOnqpmyaY8Uimnrypgln4jF+g20kdXyPq68VCSBTTE2Bgww9Kfw/HNMPNhyBwHWVOapC7sVxwvrWHy378OqK8gCIIQOkS8CEGTU1jpOK6uM7kUWvSXGddoEzqje6YA8Otp/XlhzUGvfd01xoDKTXwc/QyjdIc8+k7WNsFL00mY+k8gHtC4ufolOL0K5j0NY68L+Nm8YRdkSzbl+XV1eZu3IAiC0PKIeBECwlcNopPltWQkxFBUaUTT/AfYvv2rSeQUVpLaKRqAe2YP5vOd+S65Xuy4WHBObOPG3HuI0tVRrcXwkXkyGyzD2GYZwAnSGZNUwUfpL9Jn5c28HDWaCbp9pBir4Mzbmi1coEGQNCZcQGJe2guT+qa19RQEQWgGIl4ErxhNZh75bC+/O3cg6QkxVHmpEA1QUG6kW3IsRbZtz/7cRmN7pzK2t2uNIV/dlWaBXR/Ad8/C8c2YdfFcU/snftQG4+6cyacLXP8p1f+Zy3lFmwFYHjuLWef9vwCf1j/BBPc2pSZTUARSYEnwy3f3nkNqfHRbT0MQhGbQ3rZKC63El7tP8sZ3uTzy+V4AKt3Ey868Ml5Zf5i1+4vomtRQAbopi3d/dZwMSnFemScdesYas1JTCrMW8Vj2Yn7UhuAzqiQmgQOz3uQK45/Jqn2H5xNvb0g60wIEql8kSV340z05zqPWlSAI7QuxvEQQFotGndni84N7X0E5L6w+yKMLRjra7HEqp6tcaxTdvngrh4urAMhMiXO0B1WT6Oj3PFNzDyNi9mHRFKXfjOG96GrqNT2jj+6GcTfAnCdBp6MybztQ6XUYh1vHkMAPmjUjb0tLCEVgRo+m1GQKeiKCIAgRjlheIojHVvzE4D8vd2xvzi2pcjn/1ve5LNt2gre+z8VgM6HY4zyOl9a49LULF4ALR3R3HAe8du9eBq9eSKaWzzfmUfzXPIu46nwm6fYxWb+b42mTYPbjAVlPGnYpOVeVDnAeARKoKAm520gQBEEQy0sksWzrcQAu+NdaHrpoKD9/bRMAvzt3IHqdIinWmrtl14kyeqZarSkmi4UtR0+zfFe+1zGnZWcw0Sn4MaBFfutb8PFt0GsS1576HbtLrKKjy6x/cte7PzBJt5cZoy/lRkNDXEIggbAWH1utWwJ/w+lUw711ol4EQRBCjlheOhAllUY2Hjnl83ynGKu7KLekmie/OuBo//eqA/zzq/3sK6gAYOfxMkfNIpNFY8Fz37Js2wmiDTpevXGCy5jua3Wj4iX3O/jkDug3Da79iBoV7zilaRpGollrGYXSR7lcFkhFZ5c8MY329mTB2EymZnuvh+VPPEmciyAIQusilpcOxJyn1lNQXsvhRy/0uqAmxDT8c5fV1HucL7e15ZZUO46dtwcnxRqYMagLd8wcyL9WWsXPpeN6uYzh0/Bw6jCs+ivs/QRSesOlr0J0vIvK8MiQGySu1wd9Of+8fLTPc/7Gcz4V8pgXQRAEQcRLR2HbsVIKymsBMJq8B+UmxPoXL/Yqz2aLxn5bIjpn8WIf095287R+zBnZ3WUMr6Kj6Cdrmv7qEhh9Ncy4D+Ktribn3hanVDLuwwTiNvKVobepOFe09u82agjnFa+RIAhC6BG3UQfhcHHDTpyKWu85WTpFe4qXf13RYG04UlJNv86dANiXXw7gcB9Bg2jpZLPgjOnlmrMFvCzeuz6AF6ZAzWn42Ycw7ylI7OY47Swy/FleAnIbOfVprobYcM85fHnnNMd7fxaV2SO6BdRPEARBaBlEvHQQ6s0NC7d7ThY7zm4jO3NHduexhQ1bo++yFVI8WW61whRVGD3G/fnZfXn5uvFcMKyrx3guAas/LYcPfmmtK3TbJug1waO/M82NWXG+vrkiIjMljuS4hrgbf6OlxEXx87P7WvuJdhEEQQg54jbqIJicxYsPy0taJ9esorFROgx6nYvgGNvHak2xi5ZTVXWOc0aT1a8TbdBx3lBX4WJ3nDiGytsE798A3UbCNe9DTILXObm4jTTnrc7KrV+wbqNGuwdFY24ou9VHgncFQRBCj1heOgjOtYcqjPUUlteiad5dLXYLjN2N5GwsSbTFxdTZxquua3Ab+apv5IxSCkoOwjuXQ2JXv8LF2r/hWPPRLDiK4AAAHAhJREFUHij+xE9z8WXNshNIYcqWwB535M2KJgiCECmIeOkgOAuLg0VVTPy/Vfx71QGXPhYNEmMMjhwu8bat084LfVyU3ucC7EMLuRBlLIW3Flo7X/MBJHQJ+BlcxUfAlzXMz+k41AaQkT2TefiioY739rmH2u4yZ0R3/jRrsMO9JwiCEInI17cOgnPMS64t++07PxzFaLJw9cTeJMdH8d8Nh4nW6zDorUtstN6qXZ0XeqUUCTEGyn24nvwRQx19v/41lB+HGz6DzgMavcbZHeSSZK4JMkBr5lbrYKgzuVqh7OIl1Enq9DrFLdP7h/QegiAI4Y6Ilw6Cs+XlRJk1lX9hhZHnVx+kuMLoCD6tM1vQ2xb2KJt40bstuImxUUGLlxHqEH+N+i+d8g/Dgpeh18SArnNxGzUxT4s93qS5Ab/B4O5CswsviXkRBEEIPSJeOggmp8U077RrHaLOiTGUVjfkdbGLFbsFxj0+xFc8xV/nD/N+893LWBr1EFHKzPGpj5M58rKg5w/WwpF2miICXLZKh1hD1LmJF7twEukiCIIQeiTmpQOwI6+U/244QrReR5RecdxNvNTWmzE65WtxiBdb0UN3T0dctGeCuyU3n8l1Z2Z5ufkSWHoj+7WeXFn3AOVDrmzyc1iaaTlxTnIX6nwr9SbXACC71UjyvAiCIIQeES9hjqZp/Ow/P/Dl7gKffeY9s4FKo4kovTVepcRpezNYE9IZnWI0GsSL9dXdyhEb5fln4e5aAmDfZ/DRzdDnbK6of4jvLUODXrx9JakLoJi0By4Bu8FfHhSebiPJsCsIgtBaiHgJc+rMFtbnFHPTm5u9n3cSJQa9zqUEgJ3yGhNGkxfLi81tpHcTHDEGT8uLwX1VPvoDLP059BgLV79HNbFA8Iu3c3etHQXsPjRvGNG231NMlN5pq7SoF0EQhFAjMS9hjnPyOW84J5GL0uscuVtS4qMccS7lNfVEGxp0qs4tYNfdyuHX8qJp8MML8NVDkJwJVy+B6E6OfsGKBufuzS2s6OJ2CrGGmDeqB/VmC/llNfx6Wn8e/Hh3q9xXEARBEPES9jQmXkqqGtL3G3TKkWQuvVO0Q7yUVBldsusaGnUbebG86JW1PtHHt8G+TyF7Nsz5B3RKd+nXHLeJq/hoUoGAFplHoETpdfzhfGu+FYl5EQRBaD1EvIQ5zrtadh0v40BhBZeM6eloK6lssLzoFMTZLC/JcVEs/fWZvLL+MKt/KiLeqShjg9vIHrDruuB6W4A7FW2Dd38DFflwwaNwxi0uZgalFGhaE2JeGo6bnaQuSLfTs1ePxRxI5r0AcCSpE+0iCIIQckS8tDEHiypJiYsiPSHG63mT0xaauU+vB+Di0ZkOy4Sz5UUp5Ug8lxIfzfisNHbklfHFrgJ2Hi9z9HMP2HWPebEHoyosnKHby9m6XfT86HNI7A4/XwE9x/l8nqDFiw+R0ZSYl2DdRnNGdg/6Hr6w31osL4IgCKFHxEsbomka5z6xhi6JMfx4/0yXc8dOVdMrLd5jSy5ASVUdnW1iJ7+s1tGu00G0wbp4ptiS0vVIifO43tPy4nq+3mRmvNrHK2lvkFx1BIDqvnOIv/RZiEv1+0zNWbud87w0rTxA6+V5cachSV3r3lcQBCESEfHSCMdOVVNpNDGke1KLj330VDVgzYRrsWjUmS08900Og7sncevbW3j5uvH0y+jkcd2xU9UO8VLgLF6UcgTh2ncd2esYOeMI2PUW85K/g78cvYFuMXnUmjO4q/4mNluyeeuinxEf5zmWx9hBqg5Xt5H39sawe36aHzPTdBrcRqJeBEEQQo1slW6EKY99w+x/r3O8/8VrGxl4/+dYLJrPqs2BcshWgwjgVHUdb3x3hKe+zuHWt7dYzxdVeq3k7JxBN99NvNjdRvag20wvlheDR4Zda/s49RO8PpcorZa763/FxjkreN88nUNaD8+t0j5ozlZpSzO3OrtslQ766uahSZ4XQRCEVkPESxC8/u0RVu0rpN6s0e++z3lsxU9Bj1FpNDH0weU8v/qgSybcogqjS3FFgNT4aK+7jX777lbyTlutNs5bpRUQZdsSHWOwx75EeVyvc3MbRdeVcqfhfd6JfgTiO/NQ+hMsMc9AF5fsuMZrkjovBB3z4dTfn/jwN6y3cy0dezJ/dA+/5+2hSRLzIgiCEHpEvATBOz8cdXn/6obDQY+RU1hJdZ2Zvy/fxwPLdjnaCyuMDquJnaS4KI8aOna2HSsFoMrYUEBRqYZv/g0VoxV3zBzIogUjHP1ctkqf3MPQT+fxO8NHbLFkwy9XUqC6uPSzHgf2p9KsmBc/bh9nI9dnt0/2eq65eWL88eDcoX7P2+NtRLoIgiCEHhEvfrjItrvHjnOuFHBdUAPF1+JWVGEkSu95tt6WQfdit2/++aVWd1F1XUPmXJ1SjjnFOCWau2NmNrOGd3O8t1tRhpWthVfOQ2eu48a6u7jGdB/Ep3HvhYPpl9GJ4ZlOlhcvc/NG8LuNGgh0q3RSrKc1CUJbVbqTj2KVdqSqtCAIQush4sUPztuLAVI7uS6aLZMhxEppdR0mi+uIFk1ztKW6Cad3fzzKhpxiquvMjoy4OqUcc3JP8e/s9ulkqeRX+k+57NADkDGY/fM/4RvLGDTbn8O4Pml8/YfpLgt24DEvoQnYdX7vHhRsPxfKgN3YKD1HFs3xeV5iXgRBEFoP2W0UIBaL5ki9b6e5AbvOGE0Wl63CAGbbDiSARLdv/oeKq7jmPz8QH60nKTaK2noj1jxx1jGi3FxQdrfPJLWXO/Y9TUJUKTmpkxlw3WK0Ygtw0O/8Ao95CaibA9faRoEF7LpbqOyXaSF0GzWG1DYSBEFoPcTyEiB1ZgvV9WaXtqZoF1+XlFbX8cRX+13aLJrmCNj1VnARoKbe7CgJoFPKETiqXFUB+vzNPGZ4kbejH6FGn8xC40N8MuSfEJPo6Otv2XVPZOeO/WxzLB6WAHcLeSscCe5Vpdtmq3RTqmELgiAIwdEmH7VKqcuUUruVUhal1Hi3c/cqpXKUUj8ppS5wap9la8tRSt0T6jm6W1WM9RZq6sw+egeO2eI9ANc5zX9DX82xVTohxnecR6ItBkSnawgc1SnbyZyV8PI5RL96Hpcb1rDCMp7/DP4Pm7VBPssDeCPQ/C1BW15cdhs5j+M6UJfEhgzEMQbvf7ZagDEzocAR8yIhu4IgCCGnrb4n7gIWAGudG5VSQ4ErgWHALOA5pZReKaUHngVmA0OBq2x9Q4bRZHF7b3bZ2QONx7xUGU0e19R5yZibEGNgU+5pj3aTs3jxYXkB664ksC6c9vW7S8kmeHU2vLUQqorQ5vyTkbUv8Zv6OzDqrYnvghEvgdKcgN1aU4M4dB/mt+cOdBz7ir/R/MTMhBr7rcVrJAiCEHraRLxomrZX0zRvSVLmA4s1TTNqmnYYyAEm2n5yNE07pGlaHbDY1jdkPLfaGgOisAAat769hR8On+L30cvYEPNb3o9+mMvVKsCa8fb/fbqHspp6Pt52HIA6k4VhD61g2EMrXMb1lnQuMdbgyLYLcM5g61Zli0Vz5H5xj3lxvx6gr/kwN+Tew/LoPzHj+xvh9BG48B/w282oCb+gnATAKsQA4myBvvoW/CsIVrwkOomy8poGoec+TpRex8/O6O33HhY/lpuWZFgPz2zLUlVa+P/t3X+cVXWdx/HXZ+bO3PnBbxhQmAEEkaBgBpgAA/Mn/kAz2VTSfOS2uOZGaftwdXV1S1utR9sPNXcfWWtlmeFuWUZSqaHSaikKIgywiBYlQoKKjCgSM3z2j/O9dy537vyA4c49F9/Px+M87jnfc+65n3svcD98f4pI74lbh90RwJMZx5tDGcBLWeUzct3AzC4FLgUYOXLkQQfyjaUbqWY31yV+yIWJR3lpaw3bygcwrWRjFKi9xvtLnue52/7Eb8pP5rE/l/LEE8t40yu54t4aThxfk/O+LTmajarK2/pwXHXaeM6dVsuMLy6NVjxOddjN+JFP8leS7KXOtlPOXs5883HOKnuaOW+sZE9pNU0MZ/XEq5g870ooaz/DbvM7UZKQGsGUaro5FCN0DvQWXz2vnmk3/YbB1eU0v7O37T45rv3C2e/j+jMndtiE1RtrG228+Yycse1T8iIi0mvylryY2W+AI3Kcus7df97R03KUOblriHK22rj7t4FvAzQ2NvZoONDaigXp/UrbwzTbSFN5Pec1X8Ew28GnShfzodeXUW8Pc2XGotBN+0Zz3fN/B4wl+y3lajYq8z0kaCFBK9NZS5/l9/G1sv9l6Oa5bB4yi1H2F0atu4O7y5YwueQP9GU3JZZxn63wekkfHq04hcfqFnLPml18/eh6JudIXACad0dJQqq5qbMf3CvnHMO9T7/U4flsB/rjPbhPkrmTjuCXa/7CYxu2p8tzJVIlJUZFSe7OupDdgTo/SUT2KK6UnB2lRUQkL/KWvLj7KV1f1c5moC7juBbYEvY7Ks+LsfZyev/avQtY1HoS4+xlJo2rZ/ea19jkR3J1yye5qeUiZpU0UZ1oZVdLgoklmzi/dBk/T36OZq9kg9ex8Zt3UztmIpWjGil9azj92MVQe4PJ9gdmlqxn3q4nKKtoodWN0mXRL/ApJVX0XxOtqXRBElgOr1odv2qdznYG8KZXstUH4xiNs0/jht8203jEQE4cPhTWbOCI/hUdvrc3Q81Lv/QopY4/h8+cPG6//iZdOZgf78ykpSf3KehQ6dQMu0peRETyLm7NRouBH5nZ14HhwDhgOdF/o8eZ2VHAy0Sdei/MZyBLk1cBsMUHsXLQWbB9Nxu9lvdXVQOvpa9rpppf7ZvBl86axOtv/ZWvPLiBO1vO5MLSpUwt2Ug/e5vk1mdIvvIQ/L6VOcDqjLziHS/jp5zI5r39Sdg+Tp0zlxGTjmfqvz/JrbNbSb70OBu27+aTn7mOM768KmesU/vVAuswg8uOH0vjqIHMGDO4w/f2ZmieSc1Um6ot6cnv7tfOr+eWh5/v9mR2mc5vrOOu323ar+xgYsmseCnUaCM1G4mI5F9BkhczmwfcDtQAS8xslbuf5u5rzex/gHVAC7DQ3VvDcz4NPAiUAt9197W9Eeu8PV/gC6dP5JN3rwCikUG5jBpcxdFDow6xzVRzR+vZkDGy+uONQ1k44W22rPsdTzy3nj/5MJr2HcXLPoRmqtPXzR79AUqq+9JKKVv7vZflVXVs7vc2nx4wHMidvFSGPjNmRmmJdZq4AIwb2pcXt7+VXu6gu8OgO/PhhhF8uGFE1xfm8PkPTeSfT38PEz7363TZgcS07c09jL5mCadOHJYu6+0hy6laH6UuIiL5V6jRRj9z91p3T7r7MHc/LePcze4+1t3Hu/uvMsp/6e7HhHM35zO+1n3OTq/i+y1zeIVBjK1pSy5yrT8EUS1GRQeTpwH84JltzLh7FyuOmM9XW+bz49YTWO+jaKaar51Xn74umShJTwj3s2e3sHbLTmr6JjEzXvzi3Jz3blseoHvv76vn13PfP3yAwX2SB/S8fDEzKstL+f21J6WTw4MJ6aF1r6T3C1bzUugPU0TkXUDzgeawa+cO+tvbbPEhAFSVJ/jOxdFcevW1A3I+p08ykU4isg2saptg7qYl69ud/8i0WirLosSnPFGSnqV1/dZmtu58hwFVUQ1JR1P0p57b3SaLPskE00YNTB93NXtubzmyf2Vb8tLDmHp7gcR9WttIRKTXKHnJwbyFJ2vOY+W+qKNqRVkpJ08YxrKrTuDU9+YaQBUNZa4oa1/zctp7h7H407OZN6XzJpXUj155aUm7ZKJvJxPUpeKL7tH1L+fso4e0K2sbKt3l0/MulaD1NJbCvZcYfIgiIoc5JS859Bs0lJkL7+S0ufOAtn4uowZXd/icPhUJkh3UvNQNquKW+Q088JnZ+5V/bMZIjhsXJROpxKM8UdKuhqWr5CWlqx/sP35pLncvmN6uPPVycZjaPvXeD6Tj67KrTuCchuH7lfX2e7lt/hQumjmShrrcNXMiInLoxG20UaxcctwYLjluTLvyX15+HCv/vIPr729KlyUTpVSUtZ+ALnPukVQH2ZSb501qO0jVvCRK2jV5pEYFAXzjgilcvuhZAO65ZAZPvPAqwwdE87m8f/SgTt9PR00pcRohkxqtdCAR9asoS/ffGdInyau79nTYhJcvIwdXcdM5k7q+UEREekzJy0GYOLwff80xzX9HCwamDKxqS16G9Nk/kUn9WOfqf5JZ83J2/XBGDaqiactOZh09hFmhGeiRK49ndCc1Q52JUyfT7ta8nDC+Zr/5YWoHRgncJccdxa53WvjUiWPzF6SIiBSUkpeDNPHItvVtfvT30UoF5TlmX82ce6QyYxmAZ66fs991t184lduXbkzPepspe3h2fd0A6rOaJ8bU9Ol27NlilLt0u8/LXZ+YzuWLnmXxc1tIlpVw0cxRAFw4YyTJTkZ9iYhI8VPycpDKEyX86JIZ9Kss430j+gNRs8wNH5rIDb9Yl77Os1YD+HDDcKaOHEi244+p4fhjcq+H1LeifUJzKKVrOWKQxCTCUPTszy2Xr5w3mStPPYaq8uiP8SdmHZXP0EREJCaUvPTAB3KM3PnbWUftl7xkL8F020enHPDrZDcxHWpx6vNSGsaJ51rAMlsyUdppJ2oRETk8abRRHqRm2gWYkzHr68EaMTD3AouHSoxyl3SH3dZ9PVpTU0REDmOqecmDey6ZgQFVyQTV5T3vfzGkOtn1RT2QSl7GDCl8LUaqz0uLkhcREemAkpc8GNav4xWdD9TIQVV5Hw2UTJTynYsb23UCLgTVvIiISFeUvMTcb68+sVde5+QJPW/eOhRU8yIiIl1RnxeJldJ0zUvXHXZFROTdScmLxMpnTzmGmr5Jpo3sfLZgERF591KzkcRKQ90Anr7ulEKHISIiMaaaFxERESkqqnmJqVvnN1DTN79DpEVERIqRkpeYOmfKiEKHICIiEktqNhIREZGiouRFREREioqSFxERESkqSl5ERESkqCh5ERERkaKi5EVERESKipIXERERKSpKXkRERKSomLsXOoa8MbPtwJ8O8ulDgFcPYTj5EPcYFV/PxT1GxdczcY1vlLvXFDoIkY4c1slLT5jZM+7eWOg4OhP3GBVfz8U9RsXXM3GPTySu1GwkIiIiRUXJi4iIiBQVJS8d+3ahA+iGuMeo+Hou7jEqvp6Je3wisaQ+LyIiIlJUVPMiIiIiRUXJi4iIiBQVJS85mNnpZrbBzF4ws2sKFMN3zWybmTVllA0ys4fNbGN4HBjKzcy+EeJdbWZTeyG+OjN71MzWm9laM7sihjFWmNlyM3suxHhjKD/KzJ4KMf63mZWH8mQ4fiGcH53vGMPrlprZs2b2QNziM7NNZrbGzFaZ2TOhLE7f8QAz+4mZ/V/4s3hszOIbHz671NZsZp+NU4wixUjJSxYzKwX+EzgDmAhcYGYTCxDKXcDpWWXXAEvdfRywNBxDFOu4sF0KfLMX4msBrnT3CcBMYGH4nOIU4x7gJHevBxqA081sJvBl4JYQ4w5gQbh+AbDD3Y8GbgnX9YYrgPUZx3GL70R3b8iYjyRO3/FtwK/d/T1APdHnGJv43H1D+OwagGnA28DP4hSjSFFyd20ZG3As8GDG8bXAtQWKZTTQlHG8ATgy7B8JbAj73wIuyHVdL8b6c2BOXGMEqoCVwAyiGU0T2d838CBwbNhPhOssz3HVEv14nQQ8AFjM4tsEDMkqi8V3DPQD/pj9GcQlvhzxngo8EecYtWkrlk01L+2NAF7KON4cyuJgmLtvBQiPQ0N5QWMOzRdTgKfiFmNoklkFbAMeBl4E3nD3lhxxpGMM53cCg/Mc4q3A1cC+cDw4ZvE58JCZrTCzS0NZXL7jMcB24Huh2e1OM6uOUXzZPgosCvtxjVGkKCh5ac9ylMV9PHnBYjazPsB9wGfdvbmzS3OU5T1Gd2/1qMq+FpgOTOgkjl6N0czOAra5+4rM4k5iKMRnOMvdpxI1Zyw0sw92cm1vx5cApgLfdPcpwFu0Nb/kUsi/J+XA2cCPu7o0R1nc//0R6XVKXtrbDNRlHNcCWwoUS7ZXzOxIgPC4LZQXJGYzKyNKXO5x95/GMcYUd38DeIyof84AM0vkiCMdYzjfH3g9j2HNAs42s03AvURNR7fGKD7cfUt43EbUV2M68fmONwOb3f2pcPwTomQmLvFlOgNY6e6vhOM4xihSNJS8tPc0MC6M+CgnqupdXOCYUhYDF4f9i4n6maTKPx5GKswEdqaqpPPFzAz4DrDe3b8e0xhrzGxA2K8ETiHq0PkocG4HMaZiPxd4xN3z9r9ed7/W3WvdfTTRn7NH3P1jcYnPzKrNrG9qn6jPRhMx+Y7d/S/AS2Y2PhSdDKyLS3xZLqCtySgVS9xiFCkehe50E8cNmAs8T9Q/4roCxbAI2ArsJfrf2AKi/g1LgY3hcVC41ohGSL0IrAEaeyG+2UTV2auBVWGbG7MYJwPPhhibgM+F8jHAcuAFomr8ZCivCMcvhPNjevH7PgF4IE7xhTieC9va1N+FmH3HDcAz4Tu+HxgYp/jC61YBrwH9M8piFaM2bcW2aXkAERERKSpqNhIREZGiouRFREREioqSFxERESkqSl5ERESkqCh5ERERkaKi5EUkg5m1Zq0C3Omq4mZ2mZl9/BC87iYzG9LT+4R7VZvZw2H/8YwJ70REDgv6R01kf7s9Wk6gW9z9jnwGc5COBZ40s4HAW962TpKIyGFBNS8i3RBqRr5sZsvDdnQov8HM/insX25m68xstZndG8oGmdn9oexJM5scygeb2UNhQcFvkbGmjZldFF5jlZl9KywuWWpmd5lZk5mtMbN/zBHj2LAI5Q+BC4EVQH24z9Ds60VEipWSF5H9VWY1G83PONfs7tOB/yBagyjbNcAUd58MXBbKbgSeDWX/AvwglH8eeNyjBQUXAyMBzGwCMJ9oQcQGoBX4GNFMsiPc/X3uPgn4XvaLu/uL4TkriNYg+gGwwN0bPFqbSETksKBmI5H9ddZstCjj8ZYc51cD95jZ/URT1UO0jMJHANz9kVDj0h/4IPA3oXyJme0I158MTAOejpaPopJo0b5fAGPM7HZgCfBQJ+9hqLu/ZmaTgP/q6g2LiBQb1byIdJ93sJ9yJtG6NNOAFaGjrOW4zrMeMxnw/VBb0uDu4939BnffAdQTrYy9ELiz3RPN7jCzJqKFRVcBpwNLcjUxiYgUMyUvIt03P+Px95knzKwEqHP3R4GrgQFAH+C3RM0+mNkJwKvu3pxVfgbRgoIQLdJ3bqqPSugzMyqMRCpx9/uAfwWmZgfn7pcRNVP9G3AOsCQkQLlqiUREipaajUT2VxlqLVJ+7e6p4dJJM3uKKOm/IOt5pcAPQ5OQAbe4+xtmdgPwPTNbDbwNXByuvxFYZGYrgWXAnwHcfZ2ZXQ88FBKivUQ1LbvDfVL/4bi2g/iPJ+rrcly4r4jIYUerSot0g5ltAhrd/dVCxyIi8m6nZiMREREpKqp5ERERkaKimhcREREpKkpeREREpKgoeREREZGiouRFREREioqSFxERESkq/w93NmHNvqf7cQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print('length of scores: ', len(scores), ', len of avg_scores: ', len(avg_scores))\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores, label=\"Score\")\n",
    "plt.plot(np.arange(1, len(avg_scores)+1), avg_scores, label=\"Avg on 100 episodes\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1)) \n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episodes #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play(env, agent, num_episodes):\n",
    "    \n",
    "    state = env.reset()\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    \n",
    "    for i_episode in range(num_episodes + 1):\n",
    "        \n",
    "        state = env.reset()\n",
    "        score = 0                    \n",
    "        time_start = time.time()\n",
    "        \n",
    "        while True:\n",
    "            \n",
    "            action = agent.select_action(state, eval=True)\n",
    "            env.render()\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            score += reward \n",
    "            state = next_state\n",
    "    \n",
    "            if done:\n",
    "                break\n",
    "                \n",
    "        s = (int)(time.time() - time_start)\n",
    "        \n",
    "        scores_deque.append(score)\n",
    "        scores.append(score)    \n",
    "        \n",
    "        print('Episode {}\\tAverage Score: {:.2f},\\tScore: {:.2f} \\tTime: {:02}:{:02}:{:02}'\\\n",
    "                  .format(i_episode, np.mean(scores_deque), score, s//3600, s%3600//60, s%60)) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0\tAverage Score: 304.85,\tScore: 304.85 \tTime: 00:00:18\n",
      "Episode 1\tAverage Score: 304.79,\tScore: 304.72 \tTime: 00:00:14\n",
      "Episode 2\tAverage Score: 304.52,\tScore: 304.00 \tTime: 00:00:13\n",
      "Episode 3\tAverage Score: 304.49,\tScore: 304.39 \tTime: 00:00:14\n",
      "Episode 4\tAverage Score: 304.79,\tScore: 305.97 \tTime: 00:00:14\n",
      "Episode 5\tAverage Score: 304.74,\tScore: 304.49 \tTime: 00:00:13\n"
     ]
    }
   ],
   "source": [
    "play(env=env, agent=agent, num_episodes=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kernel-ml",
   "language": "python",
   "name": "kernel-ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
